Merge tag 's390-5.18-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux

Pull s390 updates from Vasily Gorbik:

 - Raise minimum supported machine generation to z10, which comes with
   various cleanups and code simplifications (usercopy/spectre
   mitigation/etc).

 - Rework extables and get rid of anonymous out-of-line fixups.

 - Page table helpers cleanup. Add set_pXd()/set_pte() helper functions.
   Covert pte_val()/pXd_val() macros to functions.

 - Optimize kretprobe handling by avoiding extra kprobe on
   __kretprobe_trampoline.

 - Add support for CEX8 crypto cards.

 - Allow to trigger AP bus rescan via writing to /sys/bus/ap/scans.

 - Add CONFIG_EXPOLINE_EXTERN option to build the kernel without COMDAT
   group sections which simplifies kpatch support.

 - Always use the packed stack layout and extend kernel unwinder tests.

 - Add sanity checks for ftrace code patching.

 - Add s390dbf debug log for the vfio_ap device driver.

 - Various virtual vs physical address confusion fixes.

 - Various small fixes and improvements all over the code.

* tag 's390-5.18-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux: (69 commits)
  s390/test_unwind: add kretprobe tests
  s390/kprobes: Avoid additional kprobe in kretprobe handling
  s390: convert ".insn" encoding to instruction names
  s390: assume stckf is always present
  s390/nospec: move to single register thunks
  s390: raise minimum supported machine generation to z10
  s390/uaccess: Add copy_from/to_user_key functions
  s390/nospec: align and size extern thunks
  s390/nospec: add an option to use thunk-extern
  s390/nospec: generate single register thunks if possible
  s390/pci: make zpci_set_irq()/zpci_clear_irq() static
  s390: remove unused expoline to BC instructions
  s390/irq: use assignment instead of cast
  s390/traps: get rid of magic cast for per code
  s390/traps: get rid of magic cast for program interruption code
  s390/signal: fix typo in comments
  s390/asm-offsets: remove unused defines
  s390/test_unwind: avoid build warning with W=1
  s390: remove .fixup section
  s390/bpf: encode register within extable entry
  ...

5 files changed, 222 insertions, 249 deletions

diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index 69feb8ed3312..5d415b3db6d1 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -7,6 +7,7 @@ lib-y += delay.o string.o uaccess.o find.o spinlock.o
 obj-y += mem.o xor.o
 lib-$(CONFIG_KPROBES) += probes.o
 lib-$(CONFIG_UPROBES) += probes.o
+obj-$(CONFIG_EXPOLINE_EXTERN) += expoline.o
 obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o
 test_kprobes_s390-objs += test_kprobes_asm.o test_kprobes.o
 
diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c
index bccbf394ae7e..f7f5adea8940 100644
--- a/arch/s390/lib/delay.c
+++ b/arch/s390/lib/delay.c
@@ -4,7 +4,6 @@
  *
  *    Copyright IBM Corp. 1999, 2008
  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
- *		 Heiko Carstens <heiko.carstens@de.ibm.com>,
  */
 
 #include <linux/processor.h>
diff --git a/arch/s390/lib/expoline.S b/arch/s390/lib/expoline.S
new file mode 100644
index 000000000000..92ed8409a7a4
--- /dev/null
+++ b/arch/s390/lib/expoline.S
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <asm/nospec-insn.h>
+#include <linux/linkage.h>
+
+.macro GEN_ALL_BR_THUNK_EXTERN
+	.irp r1,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+	GEN_BR_THUNK_EXTERN %r\r1
+	.endr
+.endm
+
+GEN_ALL_BR_THUNK_EXTERN
diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c
index bc7973359ae2..c01f02887de4 100644
--- a/arch/s390/lib/test_unwind.c
+++ b/arch/s390/lib/test_unwind.c
@@ -8,6 +8,7 @@
 #include <linux/completion.h>
 #include <linux/kallsyms.h>
 #include <linux/kthread.h>
+#include <linux/ftrace.h>
 #include <linux/module.h>
 #include <linux/timer.h>
 #include <linux/slab.h>
@@ -16,10 +17,14 @@
 #include <linux/wait.h>
 #include <asm/irq.h>
 
-struct kunit *current_test;
+static struct kunit *current_test;
 
 #define BT_BUF_SIZE (PAGE_SIZE * 4)
 
+static bool force_bt;
+module_param_named(backtrace, force_bt, bool, 0444);
+MODULE_PARM_DESC(backtrace, "print backtraces for all tests");
+
 /*
  * To avoid printk line limit split backtrace by lines
  */
@@ -98,7 +103,7 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs,
 		kunit_err(current_test, "Maximum number of frames exceeded\n");
 		ret = -EINVAL;
 	}
-	if (ret)
+	if (ret || force_bt)
 		print_backtrace(bt);
 	kfree(bt);
 	return ret;
@@ -124,7 +129,10 @@ static struct unwindme *unwindme;
 #define UWM_CALLER		0x8	/* Unwind starting from caller. */
 #define UWM_SWITCH_STACK	0x10	/* Use call_on_stack. */
 #define UWM_IRQ			0x20	/* Unwind from irq context. */
-#define UWM_PGM			0x40	/* Unwind from program check handler. */
+#define UWM_PGM			0x40	/* Unwind from program check handler */
+#define UWM_KPROBE_ON_FTRACE	0x80	/* Unwind from kprobe handler called via ftrace. */
+#define UWM_FTRACE		0x100	/* Unwind from ftrace handler. */
+#define UWM_KRETPROBE		0x200	/* Unwind kretprobe handlers. */
 
 static __always_inline unsigned long get_psw_addr(void)
 {
@@ -136,8 +144,56 @@ static __always_inline unsigned long get_psw_addr(void)
 	return psw_addr;
 }
 
-#ifdef CONFIG_KPROBES
-static int pgm_pre_handler(struct kprobe *p, struct pt_regs *regs)
+static int kretprobe_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
+{
+	struct unwindme *u = unwindme;
+
+	u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? regs : NULL,
+			     (u->flags & UWM_SP) ? u->sp : 0);
+
+	return 0;
+}
+
+static noinline notrace void test_unwind_kretprobed_func(void)
+{
+	asm volatile("	nop\n");
+}
+
+static noinline void test_unwind_kretprobed_func_caller(void)
+{
+	test_unwind_kretprobed_func();
+}
+
+static int test_unwind_kretprobe(struct unwindme *u)
+{
+	int ret;
+	struct kretprobe my_kretprobe;
+
+	if (!IS_ENABLED(CONFIG_KPROBES))
+		kunit_skip(current_test, "requires CONFIG_KPROBES");
+
+	u->ret = -1; /* make sure kprobe is called */
+	unwindme = u;
+
+	memset(&my_kretprobe, 0, sizeof(my_kretprobe));
+	my_kretprobe.handler = kretprobe_ret_handler;
+	my_kretprobe.maxactive = 1;
+	my_kretprobe.kp.addr = (kprobe_opcode_t *)test_unwind_kretprobed_func;
+
+	ret = register_kretprobe(&my_kretprobe);
+
+	if (ret < 0) {
+		kunit_err(current_test, "register_kretprobe failed %d\n", ret);
+		return -EINVAL;
+	}
+
+	test_unwind_kretprobed_func_caller();
+	unregister_kretprobe(&my_kretprobe);
+	unwindme = NULL;
+	return u->ret;
+}
+
+static int kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs)
 {
 	struct unwindme *u = unwindme;
 
@@ -145,8 +201,97 @@ static int pgm_pre_handler(struct kprobe *p, struct pt_regs *regs)
 			     (u->flags & UWM_SP) ? u->sp : 0);
 	return 0;
 }
+
+extern const char test_unwind_kprobed_insn[];
+
+static noinline void test_unwind_kprobed_func(void)
+{
+	asm volatile(
+		"	nopr	%%r7\n"
+		"test_unwind_kprobed_insn:\n"
+		"	nopr	%%r7\n"
+		:);
+}
+
+static int test_unwind_kprobe(struct unwindme *u)
+{
+	struct kprobe kp;
+	int ret;
+
+	if (!IS_ENABLED(CONFIG_KPROBES))
+		kunit_skip(current_test, "requires CONFIG_KPROBES");
+	if (!IS_ENABLED(CONFIG_KPROBES_ON_FTRACE) && u->flags & UWM_KPROBE_ON_FTRACE)
+		kunit_skip(current_test, "requires CONFIG_KPROBES_ON_FTRACE");
+
+	u->ret = -1; /* make sure kprobe is called */
+	unwindme = u;
+	memset(&kp, 0, sizeof(kp));
+	kp.pre_handler = kprobe_pre_handler;
+	kp.addr = u->flags & UWM_KPROBE_ON_FTRACE ?
+				(kprobe_opcode_t *)test_unwind_kprobed_func :
+				(kprobe_opcode_t *)test_unwind_kprobed_insn;
+	ret = register_kprobe(&kp);
+	if (ret < 0) {
+		kunit_err(current_test, "register_kprobe failed %d\n", ret);
+		return -EINVAL;
+	}
+
+	test_unwind_kprobed_func();
+	unregister_kprobe(&kp);
+	unwindme = NULL;
+	return u->ret;
+}
+
+static void notrace __used test_unwind_ftrace_handler(unsigned long ip,
+						      unsigned long parent_ip,
+						      struct ftrace_ops *fops,
+						      struct ftrace_regs *fregs)
+{
+	struct unwindme *u = (struct unwindme *)fregs->regs.gprs[2];
+
+	u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? &fregs->regs : NULL,
+			     (u->flags & UWM_SP) ? u->sp : 0);
+}
+
+static noinline int test_unwind_ftraced_func(struct unwindme *u)
+{
+	return READ_ONCE(u)->ret;
+}
+
+static int test_unwind_ftrace(struct unwindme *u)
+{
+	int ret;
+#ifdef CONFIG_DYNAMIC_FTRACE
+	struct ftrace_ops *fops;
+
+	fops = kunit_kzalloc(current_test, sizeof(*fops), GFP_KERNEL);
+	fops->func = test_unwind_ftrace_handler;
+	fops->flags = FTRACE_OPS_FL_DYNAMIC |
+		     FTRACE_OPS_FL_RECURSION |
+		     FTRACE_OPS_FL_SAVE_REGS |
+		     FTRACE_OPS_FL_PERMANENT;
+#else
+	kunit_skip(current_test, "requires CONFIG_DYNAMIC_FTRACE");
 #endif
 
+	ret = ftrace_set_filter_ip(fops, (unsigned long)test_unwind_ftraced_func, 0, 0);
+	if (ret) {
+		kunit_err(current_test, "failed to set ftrace filter (%d)\n", ret);
+		return -1;
+	}
+
+	ret = register_ftrace_function(fops);
+	if (!ret) {
+		ret = test_unwind_ftraced_func(u);
+		unregister_ftrace_function(fops);
+	} else {
+		kunit_err(current_test, "failed to register ftrace handler (%d)\n", ret);
+	}
+
+	ftrace_set_filter_ip(fops, (unsigned long)test_unwind_ftraced_func, 1, 0);
+	return ret;
+}
+
 /* This function may or may not appear in the backtrace. */
 static noinline int unwindme_func4(struct unwindme *u)
 {
@@ -157,35 +302,12 @@ static noinline int unwindme_func4(struct unwindme *u)
 		wait_event(u->task_wq, kthread_should_park());
 		kthread_parkme();
 		return 0;
-#ifdef CONFIG_KPROBES
-	} else if (u->flags & UWM_PGM) {
-		struct kprobe kp;
-		int ret;
-
-		unwindme = u;
-		memset(&kp, 0, sizeof(kp));
-		kp.symbol_name = "do_report_trap";
-		kp.pre_handler = pgm_pre_handler;
-		ret = register_kprobe(&kp);
-		if (ret < 0) {
-			kunit_err(current_test, "register_kprobe failed %d\n", ret);
-			return -EINVAL;
-		}
-
-		/*
-		 * Trigger operation exception; use insn notation to bypass
-		 * llvm's integrated assembler sanity checks.
-		 */
-		asm volatile(
-			"	.insn	e,0x0000\n"	/* illegal opcode */
-			"0:	nopr	%%r7\n"
-			EX_TABLE(0b, 0b)
-			:);
-
-		unregister_kprobe(&kp);
-		unwindme = NULL;
-		return u->ret;
-#endif
+	} else if (u->flags & (UWM_PGM | UWM_KPROBE_ON_FTRACE)) {
+		return test_unwind_kprobe(u);
+	} else if (u->flags & (UWM_KRETPROBE)) {
+		return test_unwind_kretprobe(u);
+	} else if (u->flags & UWM_FTRACE) {
+		return test_unwind_ftrace(u);
 	} else {
 		struct pt_regs regs;
 
@@ -255,7 +377,7 @@ static int test_unwind_irq(struct unwindme *u)
 }
 
 /* Spawns a task and passes it to test_unwind(). */
-static int test_unwind_task(struct kunit *test, struct unwindme *u)
+static int test_unwind_task(struct unwindme *u)
 {
 	struct task_struct *task;
 	int ret;
@@ -270,7 +392,7 @@ static int test_unwind_task(struct kunit *test, struct unwindme *u)
 	 */
 	task = kthread_run(unwindme_func1, u, "%s", __func__);
 	if (IS_ERR(task)) {
-		kunit_err(test, "kthread_run() failed\n");
+		kunit_err(current_test, "kthread_run() failed\n");
 		return PTR_ERR(task);
 	}
 	/*
@@ -293,49 +415,43 @@ struct test_params {
 /*
  * Create required parameter list for tests
  */
+#define TEST_WITH_FLAGS(f) { .flags = f, .name = #f }
 static const struct test_params param_list[] = {
-	{.flags = UWM_DEFAULT, .name = "UWM_DEFAULT"},
-	{.flags = UWM_SP, .name = "UWM_SP"},
-	{.flags = UWM_REGS, .name = "UWM_REGS"},
-	{.flags = UWM_SWITCH_STACK,
-		.name = "UWM_SWITCH_STACK"},
-	{.flags = UWM_SP | UWM_REGS,
-		.name = "UWM_SP | UWM_REGS"},
-	{.flags = UWM_CALLER | UWM_SP,
-		.name = "WM_CALLER | UWM_SP"},
-	{.flags = UWM_CALLER | UWM_SP | UWM_REGS,
-		.name = "UWM_CALLER | UWM_SP | UWM_REGS"},
-	{.flags = UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK,
-		.name = "UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK"},
-	{.flags = UWM_THREAD, .name = "UWM_THREAD"},
-	{.flags = UWM_THREAD | UWM_SP,
-		.name = "UWM_THREAD | UWM_SP"},
-	{.flags = UWM_THREAD | UWM_CALLER | UWM_SP,
-		.name = "UWM_THREAD | UWM_CALLER | UWM_SP"},
-	{.flags = UWM_IRQ, .name = "UWM_IRQ"},
-	{.flags = UWM_IRQ | UWM_SWITCH_STACK,
-		.name = "UWM_IRQ | UWM_SWITCH_STACK"},
-	{.flags = UWM_IRQ | UWM_SP,
-		.name = "UWM_IRQ | UWM_SP"},
-	{.flags = UWM_IRQ | UWM_REGS,
-		.name = "UWM_IRQ | UWM_REGS"},
-	{.flags = UWM_IRQ | UWM_SP | UWM_REGS,
-		.name = "UWM_IRQ | UWM_SP | UWM_REGS"},
-	{.flags = UWM_IRQ | UWM_CALLER | UWM_SP,
-		.name = "UWM_IRQ | UWM_CALLER | UWM_SP"},
-	{.flags = UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS,
-		.name = "UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS"},
-	{.flags = UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK,
-		.name = "UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK"},
-	#ifdef CONFIG_KPROBES
-	{.flags = UWM_PGM, .name = "UWM_PGM"},
-	{.flags = UWM_PGM | UWM_SP,
-		.name = "UWM_PGM | UWM_SP"},
-	{.flags = UWM_PGM | UWM_REGS,
-		.name = "UWM_PGM | UWM_REGS"},
-	{.flags = UWM_PGM | UWM_SP | UWM_REGS,
-		.name = "UWM_PGM | UWM_SP | UWM_REGS"},
-	#endif
+	TEST_WITH_FLAGS(UWM_DEFAULT),
+	TEST_WITH_FLAGS(UWM_SP),
+	TEST_WITH_FLAGS(UWM_REGS),
+	TEST_WITH_FLAGS(UWM_SWITCH_STACK),
+	TEST_WITH_FLAGS(UWM_SP | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_CALLER | UWM_SP),
+	TEST_WITH_FLAGS(UWM_CALLER | UWM_SP | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK),
+	TEST_WITH_FLAGS(UWM_THREAD),
+	TEST_WITH_FLAGS(UWM_THREAD | UWM_SP),
+	TEST_WITH_FLAGS(UWM_THREAD | UWM_CALLER | UWM_SP),
+	TEST_WITH_FLAGS(UWM_IRQ),
+	TEST_WITH_FLAGS(UWM_IRQ | UWM_SWITCH_STACK),
+	TEST_WITH_FLAGS(UWM_IRQ | UWM_SP),
+	TEST_WITH_FLAGS(UWM_IRQ | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_IRQ | UWM_SP | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP),
+	TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK),
+	TEST_WITH_FLAGS(UWM_PGM),
+	TEST_WITH_FLAGS(UWM_PGM | UWM_SP),
+	TEST_WITH_FLAGS(UWM_PGM | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_PGM | UWM_SP | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE),
+	TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_SP),
+	TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_SP | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_FTRACE),
+	TEST_WITH_FLAGS(UWM_FTRACE | UWM_SP),
+	TEST_WITH_FLAGS(UWM_FTRACE | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_FTRACE | UWM_SP | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_KRETPROBE),
+	TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP),
+	TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_REGS),
+	TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP | UWM_REGS),
 };
 
 /*
@@ -360,7 +476,7 @@ static void test_unwind_flags(struct kunit *test)
 	params = (const struct test_params *)test->param_value;
 	u.flags = params->flags;
 	if (u.flags & UWM_THREAD)
-		KUNIT_EXPECT_EQ(test, 0, test_unwind_task(test, &u));
+		KUNIT_EXPECT_EQ(test, 0, test_unwind_task(&u));
 	else if (u.flags & UWM_IRQ)
 		KUNIT_EXPECT_EQ(test, 0, test_unwind_irq(&u));
 	else
diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index b709239feb5d..d7b3b193d108 100644
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -8,13 +8,10 @@
  *		 Gerald Schaefer (gerald.schaefer@de.ibm.com)
  */
 
-#include <linux/jump_label.h>
 #include <linux/uaccess.h>
 #include <linux/export.h>
-#include <linux/errno.h>
 #include <linux/mm.h>
-#include <asm/mmu_context.h>
-#include <asm/facility.h>
+#include <asm/asm-extable.h>
 
 #ifdef CONFIG_DEBUG_ENTRY
 void debug_user_asce(int exit)
@@ -34,32 +31,8 @@ void debug_user_asce(int exit)
 }
 #endif /*CONFIG_DEBUG_ENTRY */
 
-#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
-static DEFINE_STATIC_KEY_FALSE(have_mvcos);
-
-static int __init uaccess_init(void)
-{
-	if (test_facility(27))
-		static_branch_enable(&have_mvcos);
-	return 0;
-}
-early_initcall(uaccess_init);
-
-static inline int copy_with_mvcos(void)
-{
-	if (static_branch_likely(&have_mvcos))
-		return 1;
-	return 0;
-}
-#else
-static inline int copy_with_mvcos(void)
-{
-	return 1;
-}
-#endif
-
-static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr,
-						 unsigned long size, unsigned long key)
+static unsigned long raw_copy_from_user_key(void *to, const void __user *from,
+					    unsigned long size, unsigned long key)
 {
 	unsigned long tmp1, tmp2;
 	union oac spec = {
@@ -72,7 +45,7 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr
 	tmp1 = -4096UL;
 	asm volatile(
 		"   lr	  0,%[spec]\n"
-		"0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
+		"0: mvcos 0(%2),0(%1),%0\n"
 		"6: jz    4f\n"
 		"1: algr  %0,%3\n"
 		"   slgr  %1,%3\n"
@@ -83,61 +56,18 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr
 		"   slgr  %4,%1\n"
 		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
 		"   jnh   5f\n"
-		"3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
+		"3: mvcos 0(%2),0(%1),%4\n"
 		"7: slgr  %0,%4\n"
 		"   j     5f\n"
 		"4: slgr  %0,%0\n"
 		"5:\n"
 		EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
-		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
+		: "+a" (size), "+a" (from), "+a" (to), "+a" (tmp1), "=a" (tmp2)
 		: [spec] "d" (spec.val)
 		: "cc", "memory", "0");
 	return size;
 }
 
-static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
-						unsigned long size, unsigned long key)
-{
-	unsigned long tmp1, tmp2;
-
-	tmp1 = -256UL;
-	asm volatile(
-		"   sacf  0\n"
-		"0: mvcp  0(%0,%2),0(%1),%[key]\n"
-		"7: jz    5f\n"
-		"1: algr  %0,%3\n"
-		"   la    %1,256(%1)\n"
-		"   la    %2,256(%2)\n"
-		"2: mvcp  0(%0,%2),0(%1),%[key]\n"
-		"8: jnz   1b\n"
-		"   j     5f\n"
-		"3: la    %4,255(%1)\n"	/* %4 = ptr + 255 */
-		"   lghi  %3,-4096\n"
-		"   nr    %4,%3\n"	/* %4 = (ptr + 255) & -4096 */
-		"   slgr  %4,%1\n"
-		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
-		"   jnh   6f\n"
-		"4: mvcp  0(%4,%2),0(%1),%[key]\n"
-		"9: slgr  %0,%4\n"
-		"   j     6f\n"
-		"5: slgr  %0,%0\n"
-		"6: sacf  768\n"
-		EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
-		EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
-		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
-		: [key] "d" (key << 4)
-		: "cc", "memory");
-	return size;
-}
-
-static unsigned long raw_copy_from_user_key(void *to, const void __user *from,
-					    unsigned long n, unsigned long key)
-{
-	if (copy_with_mvcos())
-		return copy_from_user_mvcos(to, from, n, key);
-	return copy_from_user_mvcp(to, from, n, key);
-}
-
 unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
 {
 	return raw_copy_from_user_key(to, from, n, 0);
@@ -160,8 +90,8 @@ unsigned long _copy_from_user_key(void *to, const void __user *from,
 }
 EXPORT_SYMBOL(_copy_from_user_key);
 
-static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
-					       unsigned long size, unsigned long key)
+static unsigned long raw_copy_to_user_key(void __user *to, const void *from,
+					  unsigned long size, unsigned long key)
 {
 	unsigned long tmp1, tmp2;
 	union oac spec = {
@@ -174,7 +104,7 @@ static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
 	tmp1 = -4096UL;
 	asm volatile(
 		"   lr	  0,%[spec]\n"
-		"0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
+		"0: mvcos 0(%1),0(%2),%0\n"
 		"6: jz    4f\n"
 		"1: algr  %0,%3\n"
 		"   slgr  %1,%3\n"
@@ -185,61 +115,18 @@ static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
 		"   slgr  %4,%1\n"
 		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
 		"   jnh   5f\n"
-		"3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
+		"3: mvcos 0(%1),0(%2),%4\n"
 		"7: slgr  %0,%4\n"
 		"   j     5f\n"
 		"4: slgr  %0,%0\n"
 		"5:\n"
 		EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
-		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
+		: "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
 		: [spec] "d" (spec.val)
 		: "cc", "memory", "0");
 	return size;
 }
 
-static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
-					      unsigned long size, unsigned long key)
-{
-	unsigned long tmp1, tmp2;
-
-	tmp1 = -256UL;
-	asm volatile(
-		"   sacf  0\n"
-		"0: mvcs  0(%0,%1),0(%2),%[key]\n"
-		"7: jz    5f\n"
-		"1: algr  %0,%3\n"
-		"   la    %1,256(%1)\n"
-		"   la    %2,256(%2)\n"
-		"2: mvcs  0(%0,%1),0(%2),%[key]\n"
-		"8: jnz   1b\n"
-		"   j     5f\n"
-		"3: la    %4,255(%1)\n" /* %4 = ptr + 255 */
-		"   lghi  %3,-4096\n"
-		"   nr    %4,%3\n"	/* %4 = (ptr + 255) & -4096 */
-		"   slgr  %4,%1\n"
-		"   clgr  %0,%4\n"	/* copy crosses next page boundary? */
-		"   jnh   6f\n"
-		"4: mvcs  0(%4,%1),0(%2),%[key]\n"
-		"9: slgr  %0,%4\n"
-		"   j     6f\n"
-		"5: slgr  %0,%0\n"
-		"6: sacf  768\n"
-		EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
-		EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
-		: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
-		: [key] "d" (key << 4)
-		: "cc", "memory");
-	return size;
-}
-
-static unsigned long raw_copy_to_user_key(void __user *to, const void *from,
-					  unsigned long n, unsigned long key)
-{
-	if (copy_with_mvcos())
-		return copy_to_user_mvcos(to, from, n, key);
-	return copy_to_user_mvcs(to, from, n, key);
-}
-
 unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 {
 	return raw_copy_to_user_key(to, from, n, 0);
@@ -257,7 +144,7 @@ unsigned long _copy_to_user_key(void __user *to, const void *from,
 }
 EXPORT_SYMBOL(_copy_to_user_key);
 
-static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
+unsigned long __clear_user(void __user *to, unsigned long size)
 {
 	unsigned long tmp1, tmp2;
 	union oac spec = {
@@ -268,7 +155,7 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size
 	tmp1 = -4096UL;
 	asm volatile(
 		"   lr	  0,%[spec]\n"
-		"0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
+		"0: mvcos 0(%1),0(%4),%0\n"
 		"   jz	  4f\n"
 		"1: algr  %0,%2\n"
 		"   slgr  %1,%2\n"
@@ -278,7 +165,7 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size
 		"   slgr  %3,%1\n"
 		"   clgr  %0,%3\n"	/* copy crosses next page boundary? */
 		"   jnh	  5f\n"
-		"3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
+		"3: mvcos 0(%1),0(%4),%3\n"
 		"   slgr  %0,%3\n"
 		"   j	  5f\n"
 		"4: slgr  %0,%0\n"
@@ -289,46 +176,4 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size
 		: "cc", "memory", "0");
 	return size;
 }
-
-static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
-{
-	unsigned long tmp1, tmp2;
-
-	asm volatile(
-		"   sacf  256\n"
-		"   aghi  %0,-1\n"
-		"   jo    5f\n"
-		"   bras  %3,3f\n"
-		"   xc    0(1,%1),0(%1)\n"
-		"0: aghi  %0,257\n"
-		"   la    %2,255(%1)\n" /* %2 = ptr + 255 */
-		"   srl   %2,12\n"
-		"   sll   %2,12\n"	/* %2 = (ptr + 255) & -4096 */
-		"   slgr  %2,%1\n"
-		"   clgr  %0,%2\n"	/* clear crosses next page boundary? */
-		"   jnh   5f\n"
-		"   aghi  %2,-1\n"
-		"1: ex    %2,0(%3)\n"
-		"   aghi  %2,1\n"
-		"   slgr  %0,%2\n"
-		"   j     5f\n"
-		"2: xc    0(256,%1),0(%1)\n"
-		"   la    %1,256(%1)\n"
-		"3: aghi  %0,-256\n"
-		"   jnm   2b\n"
-		"4: ex    %0,0(%3)\n"
-		"5: slgr  %0,%0\n"
-		"6: sacf  768\n"
-		EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
-		: "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
-		: : "cc", "memory");
-	return size;
-}
-
-unsigned long __clear_user(void __user *to, unsigned long size)
-{
-	if (copy_with_mvcos())
-			return clear_user_mvcos(to, size);
-	return clear_user_xc(to, size);
-}
 EXPORT_SYMBOL(__clear_user);