From b2d910af73e1d1d58acddcf5a47efadf34c8498d Mon Sep 17 00:00:00 2001
From: lixin <3531982712@qq.com>
Date: Mon, 27 May 2024 11:01:08 +0800
Subject: [PATCH 1/2] xenomai rt-linux patch, based on
 dovetail-core-5.10.161-on-rockchip-base-cae91899b

---
 kernel/Documentation/dovetail.rst             |   30 +
 kernel/Makefile                               |    2 +-
 kernel/arch/Kconfig                           |    3 +
 kernel/arch/arm/Kconfig                       |    9 +
 kernel/arch/arm/common/mcpm_entry.c           |   12 +-
 kernel/arch/arm/include/asm/assembler.h       |   16 +-
 kernel/arch/arm/include/asm/atomic.h          |   16 +-
 kernel/arch/arm/include/asm/bitops.h          |   24 +-
 kernel/arch/arm/include/asm/cmpxchg.h         |    8 +-
 kernel/arch/arm/include/asm/dovetail.h        |   61 +
 kernel/arch/arm/include/asm/efi.h             |    4 +
 kernel/arch/arm/include/asm/irq_pipeline.h    |  135 ++
 kernel/arch/arm/include/asm/irqflags.h        |   52 +-
 kernel/arch/arm/include/asm/mmu_context.h     |   40 +-
 kernel/arch/arm/include/asm/outercache.h      |    7 +-
 kernel/arch/arm/include/asm/ptrace.h          |    3 +
 kernel/arch/arm/include/asm/syscall.h         |    5 +
 kernel/arch/arm/include/asm/thread_info.h     |   20 +-
 .../arch/arm/include/asm/trace/exceptions.h   |   62 +
 .../arch/arm/include/asm/vdso/gettimeofday.h  |   60 +
 kernel/arch/arm/kernel/Makefile               |    5 +
 kernel/arch/arm/kernel/asm-offsets.c          |    4 +
 kernel/arch/arm/kernel/entry-armv.S           |   52 +-
 kernel/arch/arm/kernel/entry-common.S         |   79 +
 kernel/arch/arm/kernel/entry-header.S         |   23 +-
 kernel/arch/arm/kernel/irq.c                  |    9 +
 kernel/arch/arm/kernel/irq_pipeline.c         |   20 +
 kernel/arch/arm/kernel/patch.c                |    2 +-
 kernel/arch/arm/kernel/process.c              |   26 +
 kernel/arch/arm/kernel/ptrace.c               |    2 +
 kernel/arch/arm/kernel/raw_printk.c           |   30 +
 kernel/arch/arm/kernel/signal.c               |   34 +-
 kernel/arch/arm/kernel/smp.c                  |  121 +-
 kernel/arch/arm/kernel/smp_twd.c              |    7 +-
 kernel/arch/arm/kernel/traps.c                |    2 +-
 kernel/arch/arm/kernel/vdso.c                 |   51 +-
 kernel/arch/arm/mach-imx/gpc.c                |   21 +
 kernel/arch/arm/mm/alignment.c                |   21 +-
 kernel/arch/arm/mm/cache-l2x0.c               |   47 +-
 kernel/arch/arm/mm/context.c                  |   18 +-
 kernel/arch/arm/mm/fault.c                    |  109 +-
 kernel/arch/arm/vdso/datapage.S               |   11 +
 kernel/arch/arm/vfp/entry.S                   |    2 +
 kernel/arch/arm/vfp/vfphw.S                   |    2 +
 kernel/arch/arm/vfp/vfpmodule.c               |   54 +-
 kernel/arch/arm64/Kconfig                     |    4 +
 kernel/arch/arm64/boot/dts/broadcom/Makefile  |    1 +
 .../dts/broadcom/bcm2837-rpi-3-b-nobt.dts     |   12 +
 kernel/arch/arm64/include/asm/daifflags.h     |   14 +-
 kernel/arch/arm64/include/asm/dovetail.h      |   69 +
 kernel/arch/arm64/include/asm/efi.h           |    6 +
 kernel/arch/arm64/include/asm/fpsimd.h        |    1 +
 kernel/arch/arm64/include/asm/irq_pipeline.h  |  141 ++
 kernel/arch/arm64/include/asm/irqflags.h      |   46 +-
 kernel/arch/arm64/include/asm/mmu_context.h   |   34 +-
 kernel/arch/arm64/include/asm/ptrace.h        |    6 +
 kernel/arch/arm64/include/asm/syscall.h       |    5 +
 kernel/arch/arm64/include/asm/thread_info.h   |   21 +-
 kernel/arch/arm64/include/asm/uaccess.h       |    8 +-
 kernel/arch/arm64/include/asm/vdso.h          |    5 +
 .../arm64/include/asm/vdso/gettimeofday.h     |   65 +
 kernel/arch/arm64/kernel/Makefile             |    1 +
 kernel/arch/arm64/kernel/asm-offsets.c        |    1 +
 kernel/arch/arm64/kernel/debug-monitors.c     |    2 +-
 kernel/arch/arm64/kernel/entry-common.c       |  152 +-
 kernel/arch/arm64/kernel/entry.S              |   62 +-
 kernel/arch/arm64/kernel/fpsimd.c             |  196 +-
 kernel/arch/arm64/kernel/irq.c                |   10 +
 kernel/arch/arm64/kernel/irq_pipeline.c       |   23 +
 kernel/arch/arm64/kernel/process.c            |   36 +
 kernel/arch/arm64/kernel/signal.c             |   38 +-
 kernel/arch/arm64/kernel/smp.c                |  107 +-
 kernel/arch/arm64/kernel/syscall.c            |   20 +-
 kernel/arch/arm64/kernel/traps.c              |   36 +-
 kernel/arch/arm64/kernel/vdso.c               |   28 +-
 kernel/arch/arm64/kernel/vdso/vdso.lds.S      |    3 +
 kernel/arch/arm64/kernel/vdso32/vdso.lds.S    |    3 +
 kernel/arch/arm64/mm/context.c                |   11 +-
 kernel/arch/arm64/mm/fault.c                  |   40 +-
 kernel/arch/x86/Kconfig                       |    4 +
 kernel/arch/x86/entry/common.c                |   32 +
 kernel/arch/x86/entry/entry_64.S              |    5 +
 kernel/arch/x86/hyperv/hv_init.c              |    3 +-
 kernel/arch/x86/include/asm/apic.h            |    7 +-
 kernel/arch/x86/include/asm/dovetail.h        |   45 +
 kernel/arch/x86/include/asm/fpu/api.h         |   25 +-
 kernel/arch/x86/include/asm/fpu/internal.h    |   31 +-
 kernel/arch/x86/include/asm/fpu/types.h       |   12 +
 kernel/arch/x86/include/asm/i8259.h           |    2 +-
 kernel/arch/x86/include/asm/idtentry.h        |   93 +-
 kernel/arch/x86/include/asm/irq_pipeline.h    |  135 ++
 kernel/arch/x86/include/asm/irq_stack.h       |    7 +
 kernel/arch/x86/include/asm/irq_vectors.h     |   11 +-
 kernel/arch/x86/include/asm/irqflags.h        |   71 +-
 kernel/arch/x86/include/asm/mmu_context.h     |    7 +
 kernel/arch/x86/include/asm/pgtable.h         |    5 +-
 kernel/arch/x86/include/asm/special_insns.h   |    4 +-
 kernel/arch/x86/include/asm/syscall.h         |    5 +
 kernel/arch/x86/include/asm/thread_info.h     |   18 +
 kernel/arch/x86/include/asm/tlbflush.h        |    8 +-
 kernel/arch/x86/include/asm/uaccess.h         |    2 +-
 kernel/arch/x86/kernel/Makefile               |    1 +
 kernel/arch/x86/kernel/alternative.c          |   14 +-
 kernel/arch/x86/kernel/apic/apic.c            |   70 +-
 kernel/arch/x86/kernel/apic/apic_flat_64.c    |    4 +-
 kernel/arch/x86/kernel/apic/apic_numachip.c   |    4 +-
 kernel/arch/x86/kernel/apic/io_apic.c         |   85 +-
 kernel/arch/x86/kernel/apic/ipi.c             |   32 +-
 kernel/arch/x86/kernel/apic/msi.c             |   12 +-
 kernel/arch/x86/kernel/apic/vector.c          |   72 +-
 kernel/arch/x86/kernel/apic/x2apic_cluster.c  |    4 +-
 kernel/arch/x86/kernel/apic/x2apic_phys.c     |    4 +-
 kernel/arch/x86/kernel/asm-offsets.c          |    3 +
 kernel/arch/x86/kernel/cpu/acrn.c             |    3 +-
 kernel/arch/x86/kernel/cpu/mce/amd.c          |    7 +-
 kernel/arch/x86/kernel/cpu/mce/core.c         |    2 +
 kernel/arch/x86/kernel/cpu/mce/therm_throt.c  |    6 +-
 kernel/arch/x86/kernel/cpu/mce/threshold.c    |    2 +-
 kernel/arch/x86/kernel/cpu/mshyperv.c         |    6 +-
 kernel/arch/x86/kernel/cpu/mtrr/generic.c     |   12 +-
 kernel/arch/x86/kernel/dumpstack.c            |   12 +-
 kernel/arch/x86/kernel/fpu/core.c             |  111 +-
 kernel/arch/x86/kernel/fpu/signal.c           |   29 +-
 kernel/arch/x86/kernel/hpet.c                 |   10 +-
 kernel/arch/x86/kernel/i8259.c                |    3 +-
 kernel/arch/x86/kernel/idt.c                  |    4 +
 kernel/arch/x86/kernel/irq.c                  |   20 +-
 kernel/arch/x86/kernel/irq_pipeline.c         |  387 ++++
 kernel/arch/x86/kernel/irq_work.c             |    3 +-
 kernel/arch/x86/kernel/kvm.c                  |   17 +-
 kernel/arch/x86/kernel/nmi.c                  |    4 +
 kernel/arch/x86/kernel/process.c              |   16 +-
 kernel/arch/x86/kernel/process_64.c           |   32 +-
 kernel/arch/x86/kernel/smp.c                  |   15 +-
 kernel/arch/x86/kernel/smpboot.c              |    9 +-
 kernel/arch/x86/kernel/time.c                 |    2 +-
 kernel/arch/x86/kernel/traps.c                |  159 +-
 kernel/arch/x86/kernel/tsc.c                  |   19 +-
 kernel/arch/x86/kernel/tsc_sync.c             |    4 +
 kernel/arch/x86/kvm/emulate.c                 |   65 +-
 kernel/arch/x86/kvm/vmx/vmx.c                 |   37 +-
 kernel/arch/x86/kvm/x86.c                     |  106 +-
 kernel/arch/x86/lib/usercopy.c                |    2 +-
 kernel/arch/x86/mm/fault.c                    |  158 +-
 kernel/arch/x86/mm/tlb.c                      |   50 +-
 kernel/arch/x86/xen/Kconfig                   |    2 +-
 kernel/arch/x86/xen/enlighten_hvm.c           |    3 +-
 kernel/drivers/base/regmap/internal.h         |    5 +-
 kernel/drivers/base/regmap/regmap-irq.c       |    1 +
 kernel/drivers/base/regmap/regmap.c           |   41 +-
 kernel/drivers/clocksource/Kconfig            |    9 +-
 kernel/drivers/clocksource/arm_arch_timer.c   |   11 +-
 kernel/drivers/clocksource/arm_global_timer.c |   41 +-
 kernel/drivers/clocksource/bcm2835_timer.c    |   27 +-
 kernel/drivers/clocksource/clksrc_st_lpc.c    |    2 +-
 kernel/drivers/clocksource/dw_apb_timer.c     |   40 +-
 kernel/drivers/clocksource/exynos_mct.c       |   50 +-
 kernel/drivers/clocksource/mmio.c             |  503 ++++-
 kernel/drivers/clocksource/timer-imx-gpt.c    |    8 +-
 kernel/drivers/clocksource/timer-sun4i.c      |    5 +-
 .../clocksource/timer-ti-dm-systimer.c        |   29 +-
 kernel/drivers/cpuidle/cpuidle.c              |   18 +
 kernel/drivers/cpuidle/poll_state.c           |    2 +-
 kernel/drivers/dma/Kconfig                    |   18 +
 kernel/drivers/dma/bcm2835-dma.c              |  148 +-
 kernel/drivers/dma/dmaengine.c                |   10 +-
 kernel/drivers/dma/imx-sdma.c                 |  195 +-
 kernel/drivers/dma/virt-dma.c                 |  122 +-
 kernel/drivers/dma/virt-dma.h                 |  127 +-
 kernel/drivers/gpio/gpio-davinci.c            |    2 +-
 kernel/drivers/gpio/gpio-mxc.c                |    3 +-
 kernel/drivers/gpio/gpio-omap.c               |    6 +-
 kernel/drivers/gpio/gpio-pca953x.c            |    1 +
 kernel/drivers/gpio/gpio-pl061.c              |    3 +-
 kernel/drivers/gpio/gpio-xilinx.c             |   26 +-
 kernel/drivers/gpio/gpio-zynq.c               |    4 +-
 .../drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c  |    1 +
 .../drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c |    3 +-
 kernel/drivers/gpu/ipu-v3/ipu-common.c        |    1 +
 kernel/drivers/iio/industrialio-trigger.c     |    1 +
 kernel/drivers/irqchip/exynos-combiner.c      |    7 +-
 kernel/drivers/irqchip/irq-bcm2835.c          |    3 +-
 kernel/drivers/irqchip/irq-bcm2836.c          |    3 +
 kernel/drivers/irqchip/irq-gic-v2m.c          |    3 +
 .../irqchip/irq-gic-v3-its-fsl-mc-msi.c       |    3 +-
 .../drivers/irqchip/irq-gic-v3-its-pci-msi.c  |    1 +
 .../irqchip/irq-gic-v3-its-platform-msi.c     |    1 +
 kernel/drivers/irqchip/irq-gic-v3-mbi.c       |    2 +-
 kernel/drivers/irqchip/irq-gic-v3.c           |    6 +-
 kernel/drivers/irqchip/irq-gic.c              |    5 +-
 kernel/drivers/irqchip/irq-imx-irqsteer.c     |    3 +-
 kernel/drivers/irqchip/irq-omap-intc.c        |    2 +-
 kernel/drivers/irqchip/irq-sun4i.c            |    2 +-
 kernel/drivers/irqchip/irq-sunxi-nmi.c        |    4 +-
 kernel/drivers/irqchip/irq-ti-sci-inta.c      |    2 +
 kernel/drivers/memory/omap-gpmc.c             |    3 +-
 kernel/drivers/mfd/tps65217.c                 |    1 +
 .../pci/controller/dwc/pcie-designware-host.c |    1 +
 kernel/drivers/pci/controller/pcie-brcmstb.c  |    2 +
 kernel/drivers/pinctrl/bcm/pinctrl-bcm2835.c  |    4 +-
 .../pinctrl/intel/pinctrl-cherryview.c        |    5 +-
 kernel/drivers/pinctrl/qcom/pinctrl-msm.c     |    5 +-
 .../drivers/pinctrl/samsung/pinctrl-exynos.c  |   23 +-
 .../drivers/pinctrl/samsung/pinctrl-samsung.c |   22 +-
 .../drivers/pinctrl/samsung/pinctrl-samsung.h |    2 +-
 kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.c  |    5 +-
 kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.h  |    2 +-
 kernel/drivers/soc/qcom/smp2p.c               |    1 +
 kernel/drivers/soc/ti/ti_sci_inta_msi.c       |    1 +
 kernel/drivers/spi/Kconfig                    |   11 +
 kernel/drivers/spi/spi-bcm2835.c              |   99 +-
 kernel/drivers/spi/spi.c                      |  295 ++-
 kernel/drivers/spmi/spmi-pmic-arb.c           |    4 +-
 kernel/drivers/tty/serial/8250/8250_core.c    |   45 +
 kernel/drivers/tty/serial/amba-pl011.c        |   39 +
 kernel/drivers/tty/serial/imx.c               |   48 +-
 kernel/drivers/tty/serial/samsung_tty.c       |    4 +
 kernel/drivers/tty/serial/st-asc.c            |   26 +
 kernel/fs/eventfd.c                           |   10 +-
 kernel/fs/exec.c                              |   14 +
 kernel/fs/fcntl.c                             |    2 +-
 kernel/fs/file.c                              |    8 +
 kernel/fs/ioctl.c                             |   16 +
 kernel/fs/udf/inode.c                         |   76 +-
 kernel/fs/udf/truncate.c                      |   48 +-
 kernel/include/asm-generic/atomic.h           |   12 +-
 kernel/include/asm-generic/cmpxchg-local.h    |    8 +-
 kernel/include/asm-generic/cmpxchg.h          |   16 +-
 kernel/include/asm-generic/irq_pipeline.h     |  109 +
 kernel/include/asm-generic/percpu.h           |   24 +-
 kernel/include/dovetail/irq.h                 |   11 +
 kernel/include/dovetail/mm_info.h             |   12 +
 kernel/include/dovetail/netdevice.h           |   13 +
 kernel/include/dovetail/poll.h                |   12 +
 kernel/include/dovetail/spinlock.h            |   21 +
 kernel/include/dovetail/thread_info.h         |   13 +
 kernel/include/linux/clockchips.h             |   71 +-
 kernel/include/linux/clocksource.h            |   52 +
 kernel/include/linux/console.h                |    1 +
 kernel/include/linux/context_tracking_state.h |    2 +-
 kernel/include/linux/dmaengine.h              |   41 +
 kernel/include/linux/dovetail.h               |  325 +++
 kernel/include/linux/dw_apb_timer.h           |    2 +-
 kernel/include/linux/entry-common.h           |   22 +-
 kernel/include/linux/fcntl.h                  |    2 +-
 kernel/include/linux/fs.h                     |   10 +
 kernel/include/linux/hardirq.h                |   18 +
 kernel/include/linux/intel-iommu.h            |    2 +-
 kernel/include/linux/interrupt.h              |   29 +-
 kernel/include/linux/irq.h                    |   50 +-
 kernel/include/linux/irq_pipeline.h           |  145 ++
 kernel/include/linux/irqdesc.h                |   27 +-
 kernel/include/linux/irqflags.h               |   37 +
 kernel/include/linux/irqstage.h               |  398 ++++
 kernel/include/linux/kernel.h                 |    8 +-
 kernel/include/linux/kvm_host.h               |   55 +
 kernel/include/linux/lockdep.h                |  115 +-
 kernel/include/linux/mm.h                     |    1 +
 kernel/include/linux/mm_types.h               |    5 +
 kernel/include/linux/net.h                    |    1 +
 kernel/include/linux/netdevice.h              |   94 +
 kernel/include/linux/poll.h                   |    1 +
 kernel/include/linux/preempt.h                |   57 +-
 kernel/include/linux/printk.h                 |   17 +-
 kernel/include/linux/rcupdate.h               |   14 +-
 kernel/include/linux/regmap.h                 |    1 +
 kernel/include/linux/sched.h                  |   10 +
 kernel/include/linux/sched/coredump.h         |    1 +
 kernel/include/linux/skbuff.h                 |   69 +
 kernel/include/linux/smp.h                    |   15 +
 kernel/include/linux/socket.h                 |    4 +-
 kernel/include/linux/spi/spi.h                |   99 +
 kernel/include/linux/spinlock.h               |   98 +-
 kernel/include/linux/spinlock_api_up.h        |   18 +
 kernel/include/linux/spinlock_pipeline.h      |  387 ++++
 kernel/include/linux/spinlock_types.h         |  154 ++
 kernel/include/linux/stop_machine.h           |    3 +
 kernel/include/linux/thread_info.h            |   66 +
 kernel/include/linux/tick.h                   |    8 +
 kernel/include/linux/tracepoint.h             |    6 +-
 kernel/include/linux/vmalloc.h                |    1 +
 kernel/include/linux/wakeup_reason.h          |    2 +-
 kernel/include/net/netoob.h                   |   17 +
 kernel/include/net/sock.h                     |    3 +
 kernel/include/trace/events/irq.h             |   42 +
 kernel/include/uapi/asm-generic/dovetail.h    |    7 +
 kernel/include/uapi/asm-generic/fcntl.h       |    9 +
 kernel/include/uapi/linux/clocksource.h       |   33 +
 kernel/include/vdso/datapage.h                |   25 +
 kernel/init/Kconfig                           |   14 +-
 kernel/init/Makefile                          |    2 +-
 kernel/init/main.c                            |    8 +-
 kernel/kernel/Kconfig.dovetail                |   23 +
 kernel/kernel/Makefile                        |    1 +
 kernel/kernel/debug/debug_core.c              |   19 +-
 kernel/kernel/dovetail.c                      |  450 +++++
 kernel/kernel/entry/common.c                  |  196 +-
 kernel/kernel/exit.c                          |    2 +
 kernel/kernel/fork.c                          |    6 +
 kernel/kernel/irq/Kconfig                     |   13 +
 kernel/kernel/irq/Makefile                    |    2 +
 kernel/kernel/irq/chip.c                      |  269 ++-
 kernel/kernel/irq/cpuhotplug.c                |    5 +
 kernel/kernel/irq/debug.h                     |    2 +
 kernel/kernel/irq/dummychip.c                 |    4 +-
 kernel/kernel/irq/generic-chip.c              |    2 +-
 kernel/kernel/irq/handle.c                    |    9 +-
 kernel/kernel/irq/internals.h                 |    2 +
 kernel/kernel/irq/irqdesc.c                   |    9 +-
 kernel/kernel/irq/irqptorture.c               |  254 +++
 kernel/kernel/irq/manage.c                    |   89 +-
 kernel/kernel/irq/msi.c                       |    3 +
 kernel/kernel/irq/pipeline.c                  | 1764 +++++++++++++++++
 kernel/kernel/irq/proc.c                      |    3 +
 kernel/kernel/irq/resend.c                    |    8 +-
 kernel/kernel/irq/settings.h                  |   34 +
 kernel/kernel/irq_work.c                      |    9 +-
 kernel/kernel/kthread.c                       |    4 +
 kernel/kernel/locking/Makefile                |    1 +
 kernel/kernel/locking/lockdep.c               |  213 +-
 kernel/kernel/locking/lockdep_internals.h     |    4 +-
 kernel/kernel/locking/pipeline.c              |  231 +++
 kernel/kernel/locking/spinlock_debug.c        |    3 +
 kernel/kernel/notifier.c                      |    3 +
 kernel/kernel/panic.c                         |   26 +-
 kernel/kernel/power/Makefile                  |    2 +
 kernel/kernel/power/hibernate.c               |    3 +
 kernel/kernel/printk/printk.c                 |   73 +
 kernel/kernel/printk/printk_safe.c            |    3 +
 kernel/kernel/ptrace.c                        |    2 +
 kernel/kernel/rcu/tree.c                      |   31 +-
 kernel/kernel/rcu/tree_plugin.h               |    2 +-
 kernel/kernel/rcu/update.c                    |   31 +
 kernel/kernel/sched/core.c                    |  321 ++-
 kernel/kernel/sched/idle.c                    |   22 +-
 kernel/kernel/sched/sched.h                   |    2 +
 kernel/kernel/sched/wait.c                    |    2 +
 kernel/kernel/signal.c                        |   12 +-
 kernel/kernel/stop_machine.c                  |    4 +-
 kernel/kernel/time/Makefile                   |    1 +
 kernel/kernel/time/clockevents.c              |   77 +-
 kernel/kernel/time/clocksource.c              |   30 +-
 kernel/kernel/time/hrtimer.c                  |    1 +
 kernel/kernel/time/tick-broadcast.c           |   17 +
 kernel/kernel/time/tick-common.c              |   19 +-
 kernel/kernel/time/tick-internal.h            |   15 +-
 kernel/kernel/time/tick-proxy.c               |  466 +++++
 kernel/kernel/time/vsyscall.c                 |   31 +-
 kernel/kernel/trace/ftrace.c                  |   18 +-
 kernel/kernel/trace/ring_buffer.c             |   18 +-
 kernel/kernel/trace/trace.c                   |   16 +-
 kernel/kernel/trace/trace.h                   |    7 +-
 kernel/kernel/trace/trace_branch.c            |    4 +-
 kernel/kernel/trace/trace_clock.c             |    4 +-
 kernel/kernel/trace/trace_functions.c         |    4 +-
 kernel/kernel/trace/trace_functions_graph.c   |    8 +-
 kernel/kernel/trace/trace_irqsoff.c           |   11 +-
 kernel/kernel/trace/trace_output.c            |    9 +-
 kernel/kernel/trace/trace_preemptirq.c        |   52 +
 kernel/kernel/trace/trace_sched_wakeup.c      |    2 +
 kernel/kernel/trace/trace_stack.c             |    5 +-
 kernel/lib/Kconfig.debug                      |   53 +
 kernel/lib/atomic64.c                         |   24 +-
 kernel/lib/dump_stack.c                       |   36 +-
 kernel/lib/smp_processor_id.c                 |    4 +
 kernel/lib/vdso/Kconfig                       |    8 +
 kernel/lib/vdso/gettimeofday.c                |  286 ++-
 kernel/mm/ioremap.c                           |    1 +
 kernel/mm/kasan/report.c                      |    6 +-
 kernel/mm/memory.c                            |   18 +
 kernel/mm/mprotect.c                          |    4 +-
 kernel/mm/vmalloc.c                           |    6 +
 kernel/net/Kconfig                            |    3 +
 kernel/net/core/dev.c                         |   98 +-
 kernel/net/core/net-sysfs.c                   |   52 +
 kernel/net/core/skbuff.c                      |  115 ++
 kernel/net/packet/af_packet.c                 |    1 +
 kernel/net/sched/Kconfig                      |   23 +
 kernel/net/sched/Makefile                     |    1 +
 kernel/net/sched/sch_oob.c                    |  294 +++
 kernel/net/socket.c                           |  127 +-
 kernel/scripts/mkcompile_h                    |    6 +-
 kernel/security/selinux/hooks.c               |    4 +-
 kernel/security/selinux/include/classmap.h    |    4 +-
 .../perf/trace/beauty/include/linux/socket.h  |    3 +-
 385 files changed, 14720 insertions(+), 1414 deletions(-)
 create mode 100644 kernel/Documentation/dovetail.rst
 create mode 100644 kernel/arch/arm/include/asm/dovetail.h
 create mode 100644 kernel/arch/arm/include/asm/irq_pipeline.h
 create mode 100644 kernel/arch/arm/include/asm/trace/exceptions.h
 create mode 100644 kernel/arch/arm/kernel/irq_pipeline.c
 create mode 100644 kernel/arch/arm/kernel/raw_printk.c
 create mode 100644 kernel/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b-nobt.dts
 create mode 100644 kernel/arch/arm64/include/asm/dovetail.h
 create mode 100644 kernel/arch/arm64/include/asm/irq_pipeline.h
 create mode 100644 kernel/arch/arm64/kernel/irq_pipeline.c
 create mode 100644 kernel/arch/x86/include/asm/dovetail.h
 create mode 100644 kernel/arch/x86/include/asm/irq_pipeline.h
 create mode 100644 kernel/arch/x86/kernel/irq_pipeline.c
 create mode 100644 kernel/include/asm-generic/irq_pipeline.h
 create mode 100644 kernel/include/dovetail/irq.h
 create mode 100644 kernel/include/dovetail/mm_info.h
 create mode 100644 kernel/include/dovetail/netdevice.h
 create mode 100644 kernel/include/dovetail/poll.h
 create mode 100644 kernel/include/dovetail/spinlock.h
 create mode 100644 kernel/include/dovetail/thread_info.h
 create mode 100644 kernel/include/linux/dovetail.h
 create mode 100644 kernel/include/linux/irq_pipeline.h
 create mode 100644 kernel/include/linux/irqstage.h
 create mode 100644 kernel/include/linux/spinlock_pipeline.h
 create mode 100644 kernel/include/net/netoob.h
 create mode 100644 kernel/include/uapi/asm-generic/dovetail.h
 create mode 100644 kernel/include/uapi/linux/clocksource.h
 create mode 100644 kernel/kernel/Kconfig.dovetail
 create mode 100644 kernel/kernel/dovetail.c
 create mode 100644 kernel/kernel/irq/irqptorture.c
 create mode 100644 kernel/kernel/irq/pipeline.c
 create mode 100644 kernel/kernel/locking/pipeline.c
 create mode 100644 kernel/kernel/time/tick-proxy.c
 create mode 100644 kernel/net/sched/sch_oob.c

diff --git a/kernel/Documentation/dovetail.rst b/kernel/Documentation/dovetail.rst
new file mode 100644
index 0000000000..5d37b0455f
--- /dev/null
+++ b/kernel/Documentation/dovetail.rst
@@ -0,0 +1,30 @@
+========================
+Introduction to Dovetail
+========================
+
+:Author: Philippe Gerum
+:Date: 08.04.2020
+
+Using Linux as a host for lightweight software cores specialized in
+delivering very short and bounded response times has been a popular
+way of supporting real-time applications in the embedded space over
+the years.
+
+In this so-called *dual kernel* design, the time-critical work is
+immediately delegated to a small companion core running out-of-band
+with respect to the regular, in-band kernel activities. Applications
+run in user space, obtaining real-time services from the
+core. Alternatively, when there is no real-time requirement, threads
+can still use the rich GPOS feature set Linux provides such as
+networking, data storage or GUIs.
+
+*Dovetail* introduces a high-priority execution stage into the main
+kernel logic reserved for such a companion core to run on.  At any
+time, out-of-band activities from this stage can preempt the common,
+in-band work. A companion core can be implemented as as a driver,
+which connects to the main kernel via the Dovetail interface for
+delivering ultra-low latency scheduling capabilities to applications.
+
+Dovetail is fully described at https://evlproject.org/dovetail/.
+The reference implementation of a Dovetail-based companion core is
+maintained at https://evlproject.org/core/.
diff --git a/kernel/Makefile b/kernel/Makefile
index 49d1d03a89..4a2ca05ddf 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 VERSION = 5
 PATCHLEVEL = 10
-SUBLEVEL = 160
+SUBLEVEL = 161
 EXTRAVERSION =
 NAME = Dare mighty things
 
diff --git a/kernel/arch/Kconfig b/kernel/arch/Kconfig
index c3a40c57f4..42eb62fb7e 100644
--- a/kernel/arch/Kconfig
+++ b/kernel/arch/Kconfig
@@ -229,6 +229,9 @@ config HAVE_FUNCTION_ERROR_INJECTION
 config HAVE_NMI
 	bool
 
+config HAVE_PERCPU_PREEMPT_COUNT
+	bool
+
 #
 # An arch should select this if it provides all these things:
 #
diff --git a/kernel/arch/arm/Kconfig b/kernel/arch/arm/Kconfig
index f6ce22c95c..57afcf1801 100644
--- a/kernel/arch/arm/Kconfig
+++ b/kernel/arch/arm/Kconfig
@@ -236,6 +236,11 @@ config NEED_RET_TO_USER
 config ARCH_MTD_XIP
 	bool
 
+# Limited I-pipe compat (syscall routing only).
+config IPIPE_COMPAT
+	bool
+	select DOVETAIL_LEGACY_SYSCALL_RANGE
+
 config ARM_PATCH_PHYS_VIRT
 	bool "Patch physical to virtual translations at runtime" if EMBEDDED
 	default y
@@ -585,6 +590,8 @@ config ARCH_MULTI_V7
 config ARCH_MULTI_V6_V7
 	bool
 	select MIGHT_HAVE_CACHE_L2X0
+	select HAVE_IRQ_PIPELINE
+	select HAVE_DOVETAIL if CPU_HAS_ASID
 
 config ARCH_MULTI_CPU_AUTO
 	def_bool !(ARCH_MULTI_V4 || ARCH_MULTI_V4T || ARCH_MULTI_V6_V7)
@@ -1237,6 +1244,8 @@ config SCHED_SMT
 	  MultiThreading at a cost of slightly increased overhead in some
 	  places. If unsure say N here.
 
+source "kernel/Kconfig.dovetail"
+
 config HAVE_ARM_SCU
 	bool
 	help
diff --git a/kernel/arch/arm/common/mcpm_entry.c b/kernel/arch/arm/common/mcpm_entry.c
index 8a9aeeb504..53c3be5266 100644
--- a/kernel/arch/arm/common/mcpm_entry.c
+++ b/kernel/arch/arm/common/mcpm_entry.c
@@ -206,7 +206,7 @@ int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
 	 * Since this is called with IRQs enabled, and no arch_spin_lock_irq
 	 * variant exists, we need to disable IRQs manually here.
 	 */
-	local_irq_disable();
+	hard_local_irq_disable();
 	arch_spin_lock(&mcpm_lock);
 
 	cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
@@ -230,7 +230,7 @@ int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
 		ret = platform_ops->cpu_powerup(cpu, cluster);
 
 	arch_spin_unlock(&mcpm_lock);
-	local_irq_enable();
+	hard_local_irq_enable();
 	return ret;
 }
 
@@ -349,7 +349,7 @@ int mcpm_cpu_powered_up(void)
 	mpidr = read_cpuid_mpidr();
 	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
 	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	arch_spin_lock(&mcpm_lock);
 
 	cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
@@ -363,7 +363,7 @@ int mcpm_cpu_powered_up(void)
 		platform_ops->cpu_is_up(cpu, cluster);
 
 	arch_spin_unlock(&mcpm_lock);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return 0;
 }
@@ -402,7 +402,7 @@ int __init mcpm_loopback(void (*cache_disable)(void))
 	 * infrastructure. Let's play it safe by using cpu_pm_enter()
 	 * in case the CPU init code path resets the VFP or similar.
 	 */
-	local_irq_disable();
+	hard_local_irq_disable();
 	local_fiq_disable();
 	ret = cpu_pm_enter();
 	if (!ret) {
@@ -410,7 +410,7 @@ int __init mcpm_loopback(void (*cache_disable)(void))
 		cpu_pm_exit();
 	}
 	local_fiq_enable();
-	local_irq_enable();
+	hard_local_irq_enable();
 	if (ret)
 		pr_err("%s returned %d\n", __func__, ret);
 	return ret;
diff --git a/kernel/arch/arm/include/asm/assembler.h b/kernel/arch/arm/include/asm/assembler.h
index 8ff20f9b06..e2df9c4b2e 100644
--- a/kernel/arch/arm/include/asm/assembler.h
+++ b/kernel/arch/arm/include/asm/assembler.h
@@ -122,7 +122,7 @@
 	.if \save
 	stmdb   sp!, {r0-r3, ip, lr}
 	.endif
-	bl	trace_hardirqs_off
+	bl	trace_hardirqs_off_pipelined
 	.if \save
 	ldmia	sp!, {r0-r3, ip, lr}
 	.endif
@@ -138,13 +138,25 @@
 	.if \save
 	stmdb   sp!, {r0-r3, ip, lr}
 	.endif
-	bl\cond	trace_hardirqs_on
+	bl\cond	trace_hardirqs_on_pipelined
 	.if \save
 	ldmia	sp!, {r0-r3, ip, lr}
 	.endif
 #endif
 	.endm
 
+	.macro  disable_irq_if_pipelined
+#ifdef CONFIG_IRQ_PIPELINE
+	disable_irq_notrace
+#endif
+	.endm
+
+	.macro  enable_irq_if_pipelined
+#ifdef CONFIG_IRQ_PIPELINE
+	enable_irq_notrace
+#endif
+	.endm
+
 	.macro disable_irq, save=1
 	disable_irq_notrace
 	asm_trace_hardirqs_off \save
diff --git a/kernel/arch/arm/include/asm/atomic.h b/kernel/arch/arm/include/asm/atomic.h
index 455eb19a5a..0cf92e56d0 100644
--- a/kernel/arch/arm/include/asm/atomic.h
+++ b/kernel/arch/arm/include/asm/atomic.h
@@ -164,9 +164,9 @@ static inline void atomic_##op(int i, atomic_t *v)			\
 {									\
 	unsigned long flags;						\
 									\
-	raw_local_irq_save(flags);					\
+	flags = hard_local_irq_save();					\
 	v->counter c_op i;						\
-	raw_local_irq_restore(flags);					\
+	hard_local_irq_restore(flags);					\
 }									\
 
 #define ATOMIC_OP_RETURN(op, c_op, asm_op)				\
@@ -175,10 +175,10 @@ static inline int atomic_##op##_return(int i, atomic_t *v)		\
 	unsigned long flags;						\
 	int val;							\
 									\
-	raw_local_irq_save(flags);					\
+	flags = hard_local_irq_save();					\
 	v->counter c_op i;						\
 	val = v->counter;						\
-	raw_local_irq_restore(flags);					\
+	hard_local_irq_restore(flags);					\
 									\
 	return val;							\
 }
@@ -189,10 +189,10 @@ static inline int atomic_fetch_##op(int i, atomic_t *v)			\
 	unsigned long flags;						\
 	int val;							\
 									\
-	raw_local_irq_save(flags);					\
+	flags = hard_local_irq_save();					\
 	val = v->counter;						\
 	v->counter c_op i;						\
-	raw_local_irq_restore(flags);					\
+	hard_local_irq_restore(flags);					\
 									\
 	return val;							\
 }
@@ -202,11 +202,11 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
 	int ret;
 	unsigned long flags;
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	ret = v->counter;
 	if (likely(ret == old))
 		v->counter = new;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return ret;
 }
diff --git a/kernel/arch/arm/include/asm/bitops.h b/kernel/arch/arm/include/asm/bitops.h
index c92e42a5c8..9779f321b7 100644
--- a/kernel/arch/arm/include/asm/bitops.h
+++ b/kernel/arch/arm/include/asm/bitops.h
@@ -40,9 +40,9 @@ static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long *
 
 	p += BIT_WORD(bit);
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	*p |= mask;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p)
@@ -52,9 +52,9 @@ static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long
 
 	p += BIT_WORD(bit);
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	*p &= ~mask;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p)
@@ -64,9 +64,9 @@ static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned lon
 
 	p += BIT_WORD(bit);
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	*p ^= mask;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static inline int
@@ -78,10 +78,10 @@ ____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += BIT_WORD(bit);
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	res = *p;
 	*p = res | mask;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return (res & mask) != 0;
 }
@@ -95,10 +95,10 @@ ____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += BIT_WORD(bit);
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	res = *p;
 	*p = res & ~mask;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return (res & mask) != 0;
 }
@@ -112,10 +112,10 @@ ____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p)
 
 	p += BIT_WORD(bit);
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	res = *p;
 	*p = res ^ mask;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return (res & mask) != 0;
 }
diff --git a/kernel/arch/arm/include/asm/cmpxchg.h b/kernel/arch/arm/include/asm/cmpxchg.h
index 8b701f8e17..60ccad3c93 100644
--- a/kernel/arch/arm/include/asm/cmpxchg.h
+++ b/kernel/arch/arm/include/asm/cmpxchg.h
@@ -77,17 +77,17 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
 #error SMP is not supported on this platform
 #endif
 	case 1:
-		raw_local_irq_save(flags);
+		flags = hard_local_irq_save();
 		ret = *(volatile unsigned char *)ptr;
 		*(volatile unsigned char *)ptr = x;
-		raw_local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		break;
 
 	case 4:
-		raw_local_irq_save(flags);
+		flags = hard_local_irq_save();
 		ret = *(volatile unsigned long *)ptr;
 		*(volatile unsigned long *)ptr = x;
-		raw_local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		break;
 #else
 	case 1:
diff --git a/kernel/arch/arm/include/asm/dovetail.h b/kernel/arch/arm/include/asm/dovetail.h
new file mode 100644
index 0000000000..f8fe64a18e
--- /dev/null
+++ b/kernel/arch/arm/include/asm/dovetail.h
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum.
+ */
+#ifndef _ASM_ARM_DOVETAIL_H
+#define _ASM_ARM_DOVETAIL_H
+
+/* ARM traps */
+#define ARM_TRAP_ACCESS		0	/* Data or instruction access exception */
+#define ARM_TRAP_SECTION	1	/* Section fault */
+#define ARM_TRAP_DABT		2	/* Generic data abort */
+#define ARM_TRAP_PABT		3	/* Prefetch abort */
+#define ARM_TRAP_BREAK		4	/* Instruction breakpoint */
+#define ARM_TRAP_FPU		5	/* Floating point exception */
+#define ARM_TRAP_VFP		6	/* VFP floating point exception */
+#define ARM_TRAP_UNDEFINSTR	7	/* Undefined instruction */
+#define ARM_TRAP_ALIGNMENT	8	/* Unaligned access exception */
+
+#if !defined(__ASSEMBLY__)
+
+#ifdef CONFIG_DOVETAIL
+
+static inline void arch_dovetail_exec_prepare(void)
+{ }
+
+static inline void arch_dovetail_switch_prepare(bool leave_inband)
+{ }
+
+static inline void arch_dovetail_switch_finish(bool enter_inband)
+{ }
+
+#endif
+
+/*
+ * Pass the trap event to the companion core. Return true if running
+ * in-band afterwards.
+ */
+#define mark_cond_trap_entry(__trapnr, __regs)		\
+	({						\
+		oob_trap_notify(__trapnr, __regs);	\
+		running_inband();			\
+	})
+
+/*
+ * Pass the trap event to the companion core. We expect the current
+ * context to be running on the in-band stage upon return so that our
+ * caller can tread on common kernel code.
+ */
+#define mark_trap_entry(__trapnr, __regs)				\
+	do {								\
+		bool __ret = mark_cond_trap_entry(__trapnr, __regs);	\
+		BUG_ON(dovetail_debug() && !__ret);			\
+	} while (0)
+
+#define mark_trap_exit(__trapnr, __regs)				\
+	oob_trap_unwind(__trapnr, __regs)
+
+#endif	/* !__ASSEMBLY__ */
+
+#endif /* _ASM_ARM_DOVETAIL_H */
diff --git a/kernel/arch/arm/include/asm/efi.h b/kernel/arch/arm/include/asm/efi.h
index 3ee4f43819..dedafc6f7e 100644
--- a/kernel/arch/arm/include/asm/efi.h
+++ b/kernel/arch/arm/include/asm/efi.h
@@ -37,7 +37,11 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md);
 
 static inline void efi_set_pgd(struct mm_struct *mm)
 {
+	unsigned long flags;
+
+	protect_inband_mm(flags);
 	check_and_switch_context(mm, NULL);
+	unprotect_inband_mm(flags);
 }
 
 void efi_virtmap_load(void);
diff --git a/kernel/arch/arm/include/asm/irq_pipeline.h b/kernel/arch/arm/include/asm/irq_pipeline.h
new file mode 100644
index 0000000000..5970c6d86d
--- /dev/null
+++ b/kernel/arch/arm/include/asm/irq_pipeline.h
@@ -0,0 +1,135 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#ifndef _ASM_ARM_IRQ_PIPELINE_H
+#define _ASM_ARM_IRQ_PIPELINE_H
+
+#include <asm-generic/irq_pipeline.h>
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+/*
+ * In order to cope with the limited number of SGIs available to us,
+ * In-band IPI messages are multiplexed over SGI0, whereas out-of-band
+ * IPIs are directly mapped to SGI1-2.
+ */
+#define OOB_NR_IPI		2
+#define OOB_IPI_OFFSET		1 /* SGI1 */
+#define TIMER_OOB_IPI		(ipi_irq_base + OOB_IPI_OFFSET)
+#define RESCHEDULE_OOB_IPI	(TIMER_OOB_IPI + 1)
+
+extern int ipi_irq_base;
+
+static inline notrace
+unsigned long arch_irqs_virtual_to_native_flags(int stalled)
+{
+	return (!!stalled) << IRQMASK_I_POS;
+}
+
+static inline notrace
+unsigned long arch_irqs_native_to_virtual_flags(unsigned long flags)
+{
+	return (!!hard_irqs_disabled_flags(flags)) << IRQMASK_i_POS;
+}
+
+static inline notrace unsigned long arch_local_irq_save(void)
+{
+	int stalled = inband_irq_save();
+	barrier();
+	return arch_irqs_virtual_to_native_flags(stalled);
+}
+
+static inline notrace void arch_local_irq_enable(void)
+{
+	barrier();
+	inband_irq_enable();
+}
+
+static inline notrace void arch_local_irq_disable(void)
+{
+	inband_irq_disable();
+	barrier();
+}
+
+static inline notrace unsigned long arch_local_save_flags(void)
+{
+	int stalled = inband_irqs_disabled();
+	barrier();
+	return arch_irqs_virtual_to_native_flags(stalled);
+}
+
+static inline int arch_irqs_disabled_flags(unsigned long flags)
+{
+	return native_irqs_disabled_flags(flags);
+}
+
+static inline notrace void arch_local_irq_restore(unsigned long flags)
+{
+	inband_irq_restore(arch_irqs_disabled_flags(flags));
+	barrier();
+}
+
+static inline
+void arch_save_timer_regs(struct pt_regs *dst, struct pt_regs *src)
+{
+	dst->ARM_cpsr = src->ARM_cpsr;
+	dst->ARM_pc = src->ARM_pc;
+}
+
+static inline bool arch_steal_pipelined_tick(struct pt_regs *regs)
+{
+	return !!(regs->ARM_cpsr & IRQMASK_I_BIT);
+}
+
+static inline int arch_enable_oob_stage(void)
+{
+	return 0;
+}
+
+#define arch_kentry_get_irqstate(__regs)		\
+	({						\
+		to_svc_pt_regs(__regs)->irqstate;	\
+	})
+
+#define arch_kentry_set_irqstate(__regs, __irqstate)		\
+	do {							\
+		to_svc_pt_regs(__regs)->irqstate = __irqstate;	\
+	} while (0)
+
+#else /* !CONFIG_IRQ_PIPELINE */
+
+static inline unsigned long arch_local_irq_save(void)
+{
+	return native_irq_save();
+}
+
+static inline void arch_local_irq_enable(void)
+{
+	native_irq_enable();
+}
+
+static inline void arch_local_irq_disable(void)
+{
+	native_irq_disable();
+}
+
+static inline unsigned long arch_local_save_flags(void)
+{
+	return native_save_flags();
+}
+
+static inline void arch_local_irq_restore(unsigned long flags)
+{
+	native_irq_restore(flags);
+}
+
+static inline int arch_irqs_disabled_flags(unsigned long flags)
+{
+	return native_irqs_disabled_flags(flags);
+}
+
+#endif /* !CONFIG_IRQ_PIPELINE */
+
+#endif /* _ASM_ARM_IRQ_PIPELINE_H */
diff --git a/kernel/arch/arm/include/asm/irqflags.h b/kernel/arch/arm/include/asm/irqflags.h
index aeec7f24eb..3c67ce236f 100644
--- a/kernel/arch/arm/include/asm/irqflags.h
+++ b/kernel/arch/arm/include/asm/irqflags.h
@@ -5,6 +5,7 @@
 #ifdef __KERNEL__
 
 #include <asm/ptrace.h>
+#include <asm/barrier.h>
 
 /*
  * CPU interrupt mask handling.
@@ -13,41 +14,44 @@
 #define IRQMASK_REG_NAME_R "primask"
 #define IRQMASK_REG_NAME_W "primask"
 #define IRQMASK_I_BIT	1
+#define IRQMASK_I_POS	0
 #else
 #define IRQMASK_REG_NAME_R "cpsr"
 #define IRQMASK_REG_NAME_W "cpsr_c"
 #define IRQMASK_I_BIT	PSR_I_BIT
+#define IRQMASK_I_POS	7
 #endif
+#define IRQMASK_i_POS	31
 
 #if __LINUX_ARM_ARCH__ >= 6
 
 #define arch_local_irq_save arch_local_irq_save
-static inline unsigned long arch_local_irq_save(void)
+static inline unsigned long native_irq_save(void)
 {
 	unsigned long flags;
 
 	asm volatile(
-		"	mrs	%0, " IRQMASK_REG_NAME_R "	@ arch_local_irq_save\n"
+		"	mrs	%0, " IRQMASK_REG_NAME_R "	@ native_irq_save\n"
 		"	cpsid	i"
 		: "=r" (flags) : : "memory", "cc");
 	return flags;
 }
 
 #define arch_local_irq_enable arch_local_irq_enable
-static inline void arch_local_irq_enable(void)
+static inline void native_irq_enable(void)
 {
 	asm volatile(
-		"	cpsie i			@ arch_local_irq_enable"
+		"	cpsie i			@ native_irq_enable"
 		:
 		:
 		: "memory", "cc");
 }
 
 #define arch_local_irq_disable arch_local_irq_disable
-static inline void arch_local_irq_disable(void)
+static inline void native_irq_disable(void)
 {
 	asm volatile(
-		"	cpsid i			@ arch_local_irq_disable"
+		"	cpsid i			@ native_irq_disable"
 		:
 		:
 		: "memory", "cc");
@@ -69,12 +73,12 @@ static inline void arch_local_irq_disable(void)
  * Save the current interrupt enable state & disable IRQs
  */
 #define arch_local_irq_save arch_local_irq_save
-static inline unsigned long arch_local_irq_save(void)
+static inline unsigned long native_irq_save(void)
 {
 	unsigned long flags, temp;
 
 	asm volatile(
-		"	mrs	%0, cpsr	@ arch_local_irq_save\n"
+		"	mrs	%0, cpsr	@ native_irq_save\n"
 		"	orr	%1, %0, #128\n"
 		"	msr	cpsr_c, %1"
 		: "=r" (flags), "=r" (temp)
@@ -87,11 +91,11 @@ static inline unsigned long arch_local_irq_save(void)
  * Enable IRQs
  */
 #define arch_local_irq_enable arch_local_irq_enable
-static inline void arch_local_irq_enable(void)
+static inline void native_irq_enable(void)
 {
 	unsigned long temp;
 	asm volatile(
-		"	mrs	%0, cpsr	@ arch_local_irq_enable\n"
+		"	mrs	%0, cpsr	@ native_irq_enable\n"
 		"	bic	%0, %0, #128\n"
 		"	msr	cpsr_c, %0"
 		: "=r" (temp)
@@ -103,11 +107,11 @@ static inline void arch_local_irq_enable(void)
  * Disable IRQs
  */
 #define arch_local_irq_disable arch_local_irq_disable
-static inline void arch_local_irq_disable(void)
+static inline void native_irq_disable(void)
 {
 	unsigned long temp;
 	asm volatile(
-		"	mrs	%0, cpsr	@ arch_local_irq_disable\n"
+		"	mrs	%0, cpsr	@ native_irq_disable\n"
 		"	orr	%0, %0, #128\n"
 		"	msr	cpsr_c, %0"
 		: "=r" (temp)
@@ -149,15 +153,22 @@ static inline void arch_local_irq_disable(void)
 #define local_abt_disable()	do { } while (0)
 #endif
 
+static inline void native_irq_sync(void)
+{
+	native_irq_enable();
+	isb();
+	native_irq_disable();
+}
+
 /*
  * Save the current interrupt enable state.
  */
 #define arch_local_save_flags arch_local_save_flags
-static inline unsigned long arch_local_save_flags(void)
+static inline unsigned long native_save_flags(void)
 {
 	unsigned long flags;
 	asm volatile(
-		"	mrs	%0, " IRQMASK_REG_NAME_R "	@ local_save_flags"
+		"	mrs	%0, " IRQMASK_REG_NAME_R "	@ native_save_flags"
 		: "=r" (flags) : : "memory", "cc");
 	return flags;
 }
@@ -166,21 +177,28 @@ static inline unsigned long arch_local_save_flags(void)
  * restore saved IRQ & FIQ state
  */
 #define arch_local_irq_restore arch_local_irq_restore
-static inline void arch_local_irq_restore(unsigned long flags)
+static inline void native_irq_restore(unsigned long flags)
 {
 	asm volatile(
-		"	msr	" IRQMASK_REG_NAME_W ", %0	@ local_irq_restore"
+		"	msr	" IRQMASK_REG_NAME_W ", %0	@ native_irq_restore"
 		:
 		: "r" (flags)
 		: "memory", "cc");
 }
 
 #define arch_irqs_disabled_flags arch_irqs_disabled_flags
-static inline int arch_irqs_disabled_flags(unsigned long flags)
+static inline int native_irqs_disabled_flags(unsigned long flags)
 {
 	return flags & IRQMASK_I_BIT;
 }
 
+static inline bool native_irqs_disabled(void)
+{
+	unsigned long flags = native_save_flags();
+	return native_irqs_disabled_flags(flags);
+}
+
+#include <asm/irq_pipeline.h>
 #include <asm-generic/irqflags.h>
 
 #endif /* ifdef __KERNEL__ */
diff --git a/kernel/arch/arm/include/asm/mmu_context.h b/kernel/arch/arm/include/asm/mmu_context.h
index f99ed524fe..32cded3e53 100644
--- a/kernel/arch/arm/include/asm/mmu_context.h
+++ b/kernel/arch/arm/include/asm/mmu_context.h
@@ -14,6 +14,7 @@
 #include <linux/sched.h>
 #include <linux/mm_types.h>
 #include <linux/preempt.h>
+#include <linux/irq_pipeline.h>
 
 #include <asm/cacheflush.h>
 #include <asm/cachetype.h>
@@ -72,6 +73,7 @@ static inline void check_and_switch_context(struct mm_struct *mm,
 static inline void finish_arch_post_lock_switch(void)
 {
 	struct mm_struct *mm = current->mm;
+	unsigned long flags;
 
 	if (mm && mm->context.switch_pending) {
 		/*
@@ -83,7 +85,9 @@ static inline void finish_arch_post_lock_switch(void)
 		preempt_disable();
 		if (mm->context.switch_pending) {
 			mm->context.switch_pending = 0;
+			protect_inband_mm(flags);
 			cpu_switch_mm(mm->pgd, mm);
+			unprotect_inband_mm(flags);
 		}
 		preempt_enable_no_resched();
 	}
@@ -102,7 +106,7 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 #endif	/* CONFIG_CPU_HAS_ASID */
 
 #define destroy_context(mm)		do { } while(0)
-#define activate_mm(prev,next)		switch_mm(prev, next, NULL)
+#define activate_mm(prev,next)		__switch_mm(prev, next, NULL)
 
 /*
  * This is called when "tsk" is about to enter lazy TLB mode.
@@ -118,15 +122,9 @@ enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
 }
 
-/*
- * This is the actual mm switch as far as the scheduler
- * is concerned.  No registers are touched.  We avoid
- * calling the CPU specific function when the mm hasn't
- * actually changed.
- */
 static inline void
-switch_mm(struct mm_struct *prev, struct mm_struct *next,
-	  struct task_struct *tsk)
+__switch_mm(struct mm_struct *prev, struct mm_struct *next,
+	    struct task_struct *tsk)
 {
 #ifdef CONFIG_MMU
 	unsigned int cpu = smp_processor_id();
@@ -149,6 +147,30 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
 #endif
 }
 
+/*
+ * This is the actual mm switch as far as the scheduler
+ * is concerned.  No registers are touched.  We avoid
+ * calling the CPU specific function when the mm hasn't
+ * actually changed.
+ */
+static inline void
+switch_mm(struct mm_struct *prev, struct mm_struct *next,
+	  struct task_struct *tsk)
+{
+	unsigned long flags;
+
+	protect_inband_mm(flags);
+	__switch_mm(prev, next, tsk);
+	unprotect_inband_mm(flags);
+}
+
 #define deactivate_mm(tsk,mm)	do { } while (0)
 
+static inline void
+switch_oob_mm(struct mm_struct *prev, struct mm_struct *next,
+	      struct task_struct *tsk)
+{
+	__switch_mm(prev, next, tsk);
+}
+
 #endif
diff --git a/kernel/arch/arm/include/asm/outercache.h b/kernel/arch/arm/include/asm/outercache.h
index 3364637755..811978d36e 100644
--- a/kernel/arch/arm/include/asm/outercache.h
+++ b/kernel/arch/arm/include/asm/outercache.h
@@ -78,8 +78,13 @@ static inline void outer_flush_range(phys_addr_t start, phys_addr_t end)
  */
 static inline void outer_flush_all(void)
 {
-	if (outer_cache.flush_all)
+	unsigned long flags;
+
+	if (outer_cache.flush_all) {
+		flags = hard_cond_local_irq_save();
 		outer_cache.flush_all();
+		hard_cond_local_irq_restore(flags);
+	}
 }
 
 /**
diff --git a/kernel/arch/arm/include/asm/ptrace.h b/kernel/arch/arm/include/asm/ptrace.h
index 73c83f4d33..5b4f5c4e99 100644
--- a/kernel/arch/arm/include/asm/ptrace.h
+++ b/kernel/arch/arm/include/asm/ptrace.h
@@ -20,6 +20,9 @@ struct svc_pt_regs {
 	struct pt_regs regs;
 	u32 dacr;
 	u32 addr_limit;
+#ifdef CONFIG_IRQ_PIPELINE
+	long irqstate;
+#endif
 };
 
 #define to_svc_pt_regs(r) container_of(r, struct svc_pt_regs, regs)
diff --git a/kernel/arch/arm/include/asm/syscall.h b/kernel/arch/arm/include/asm/syscall.h
index fd02761ba0..eec5bca5cb 100644
--- a/kernel/arch/arm/include/asm/syscall.h
+++ b/kernel/arch/arm/include/asm/syscall.h
@@ -63,6 +63,11 @@ static inline void syscall_get_arguments(struct task_struct *task,
 	memcpy(args, &regs->ARM_r0 + 1, 5 * sizeof(args[0]));
 }
 
+static inline unsigned long syscall_get_arg0(struct pt_regs *regs)
+{
+	return regs->ARM_ORIG_r0;
+}
+
 static inline void syscall_set_arguments(struct task_struct *task,
 					 struct pt_regs *regs,
 					 const unsigned long *args)
diff --git a/kernel/arch/arm/include/asm/thread_info.h b/kernel/arch/arm/include/asm/thread_info.h
index eb7ce2747e..7c84af4875 100644
--- a/kernel/arch/arm/include/asm/thread_info.h
+++ b/kernel/arch/arm/include/asm/thread_info.h
@@ -21,6 +21,7 @@
 
 struct task_struct;
 
+#include <dovetail/thread_info.h>
 #include <asm/types.h>
 
 typedef unsigned long mm_segment_t;
@@ -45,6 +46,7 @@ struct cpu_context_save {
  */
 struct thread_info {
 	unsigned long		flags;		/* low level flags */
+	__u32			local_flags;	/* local (synchronous) flags */
 	int			preempt_count;	/* 0 => preemptable, <0 => bug */
 	mm_segment_t		addr_limit;	/* address limit */
 	struct task_struct	*task;		/* main task structure */
@@ -65,16 +67,20 @@ struct thread_info {
 #ifdef CONFIG_ARM_THUMBEE
 	unsigned long		thumbee_state;	/* ThumbEE Handler Base register */
 #endif
+	struct oob_thread_state	oob_state; /* co-kernel thread state */
 };
 
 #define INIT_THREAD_INFO(tsk)						\
 {									\
 	.task		= &tsk,						\
 	.flags		= 0,						\
+	.local_flags	= 0,						\
 	.preempt_count	= INIT_PREEMPT_COUNT,				\
 	.addr_limit	= KERNEL_DS,					\
 }
 
+#define ti_local_flags(__ti)	((__ti)->local_flags)
+
 /*
  * how to get the thread information struct from C
  */
@@ -142,6 +148,8 @@ extern int vfp_restore_user_hwstate(struct user_vfp *,
 #define TIF_USING_IWMMXT	17
 #define TIF_MEMDIE		18	/* is terminating due to OOM killer */
 #define TIF_RESTORE_SIGMASK	20
+#define TIF_MAYDAY		21	/* emergency trap pending */
+#define TIF_RETUSER		22	/* INBAND_TASK_RETUSER is pending */
 
 #define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
@@ -153,6 +161,8 @@ extern int vfp_restore_user_hwstate(struct user_vfp *,
 #define _TIF_SECCOMP		(1 << TIF_SECCOMP)
 #define _TIF_NOTIFY_SIGNAL	(1 << TIF_NOTIFY_SIGNAL)
 #define _TIF_USING_IWMMXT	(1 << TIF_USING_IWMMXT)
+#define _TIF_MAYDAY		(1 << TIF_MAYDAY)
+#define _TIF_RETUSER		(1 << TIF_RETUSER)
 
 /* Checks for any syscall work in entry-common.S */
 #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
@@ -163,7 +173,15 @@ extern int vfp_restore_user_hwstate(struct user_vfp *,
  */
 #define _TIF_WORK_MASK		(_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
 				 _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
-				 _TIF_NOTIFY_SIGNAL)
+				 _TIF_NOTIFY_SIGNAL | _TIF_RETUSER)
+
+/*
+ * Local (synchronous) thread flags.
+ */
+#define _TLF_OOB		0x0001
+#define _TLF_DOVETAIL		0x0002
+#define _TLF_OFFSTAGE		0x0004
+#define _TLF_OOBTRAP		0x0008
 
 #endif /* __KERNEL__ */
 #endif /* __ASM_ARM_THREAD_INFO_H */
diff --git a/kernel/arch/arm/include/asm/trace/exceptions.h b/kernel/arch/arm/include/asm/trace/exceptions.h
new file mode 100644
index 0000000000..bdb666b3da
--- /dev/null
+++ b/kernel/arch/arm/include/asm/trace/exceptions.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM exceptions
+
+#if !defined(_TRACE_EXCEPTIONS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_EXCEPTIONS_H
+
+#include <linux/tracepoint.h>
+#include <asm/ptrace.h>
+#include <asm/dovetail.h>
+
+#define __trace_trap(__sym)	{ __sym, #__sym }
+
+#define trace_trap_symbolic(__trapnr)				\
+	__print_symbolic(__trapnr,				\
+			__trace_trap(ARM_TRAP_ACCESS),		\
+			__trace_trap(ARM_TRAP_SECTION),		\
+			__trace_trap(ARM_TRAP_DABT),		\
+			__trace_trap(ARM_TRAP_PABT),		\
+			__trace_trap(ARM_TRAP_BREAK),		\
+			__trace_trap(ARM_TRAP_FPU),		\
+			__trace_trap(ARM_TRAP_VFP),		\
+			__trace_trap(ARM_TRAP_UNDEFINSTR),	\
+			__trace_trap(ARM_TRAP_ALIGNMENT))
+
+DECLARE_EVENT_CLASS(ARM_trap_event,
+	TP_PROTO(int trapnr, struct pt_regs *regs),
+	TP_ARGS(trapnr, regs),
+
+	TP_STRUCT__entry(
+		__field(int, trapnr)
+		__field(struct pt_regs *, regs)
+		),
+
+	TP_fast_assign(
+		__entry->trapnr = trapnr;
+		__entry->regs = regs;
+		),
+
+	TP_printk("%s mode trap: %s",
+		user_mode(__entry->regs) ? "user" : "kernel",
+		trace_trap_symbolic(__entry->trapnr))
+);
+
+DEFINE_EVENT(ARM_trap_event, ARM_trap_entry,
+	TP_PROTO(int trapnr, struct pt_regs *regs),
+	TP_ARGS(trapnr, regs)
+);
+
+DEFINE_EVENT(ARM_trap_event, ARM_trap_exit,
+	TP_PROTO(int trapnr, struct pt_regs *regs),
+	TP_ARGS(trapnr, regs)
+);
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH asm/trace
+#define TRACE_INCLUDE_FILE exceptions
+#endif /*  _TRACE_EXCEPTIONS_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/kernel/arch/arm/include/asm/vdso/gettimeofday.h b/kernel/arch/arm/include/asm/vdso/gettimeofday.h
index 2134cbd546..eadbcde367 100644
--- a/kernel/arch/arm/include/asm/vdso/gettimeofday.h
+++ b/kernel/arch/arm/include/asm/vdso/gettimeofday.h
@@ -142,6 +142,66 @@ static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
 	return __get_datapage();
 }
 
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+
+extern struct vdso_priv *__get_privpage(void);
+
+static __always_inline struct vdso_priv *__arch_get_vdso_priv(void)
+{
+	return __get_privpage();
+}
+
+static __always_inline long clock_open_device(const char *path, int mode)
+{
+	register u32 r0 asm("r0") = (u32)path;
+	register u32 r1 asm("r1") = (u32)mode;
+	register long ret asm ("r0");
+	register long nr asm("r7") = __NR_open;
+
+	asm volatile(
+		"	swi #0\n"
+		: "=r" (ret)
+		: "r"(r0), "r"(r1), "r"(nr)
+		: "memory");
+
+	return ret;
+}
+
+static __always_inline
+long clock_ioctl_device(int fd, unsigned int cmd, long arg)
+{
+	register u32 r0 asm("r0") = (u32)fd;
+	register u32 r1 asm("r1") = (u32)cmd;
+	register u32 r2 asm("r2") = (u32)arg;
+	register long ret asm ("r0");
+	register long nr asm("r7") = __NR_ioctl;
+
+ 	asm volatile(
+		"	swi #0\n"
+		: "=r" (ret)
+		: "r"(r0), "r"(r1), "r"(r2), "r"(nr)
+		: "memory");
+
+ 	return ret;
+}
+
+static __always_inline long clock_close_device(int fd)
+{
+	register u32 r0 asm("r0") = (u32)fd;
+	register long ret asm ("r0");
+	register long nr asm("r7") = __NR_close;
+
+	asm volatile(
+		"	swi #0\n"
+		: "=r" (ret)
+		: "r"(r0), "r"(nr)
+		: "memory");
+
+	return ret;
+}
+
+#endif	/* CONFIG_GENERIC_CLOCKSOURCE_VDSO */
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* __ASM_VDSO_GETTIMEOFDAY_H */
diff --git a/kernel/arch/arm/kernel/Makefile b/kernel/arch/arm/kernel/Makefile
index 79588b5623..5d9267c094 100644
--- a/kernel/arch/arm/kernel/Makefile
+++ b/kernel/arch/arm/kernel/Makefile
@@ -92,6 +92,11 @@ obj-$(CONFIG_PARAVIRT)	+= paravirt.o
 head-y			:= head$(MMUEXT).o
 obj-$(CONFIG_DEBUG_LL)	+= debug.o
 obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
+ifeq ($(CONFIG_DEBUG_LL),y)
+obj-$(CONFIG_RAW_PRINTK)	+= raw_printk.o
+endif
+
+obj-$(CONFIG_IRQ_PIPELINE)	+= irq_pipeline.o
 
 # This is executed very early using a temporary stack when no memory allocator
 # nor global data is available. Everything has to be allocated on the stack.
diff --git a/kernel/arch/arm/kernel/asm-offsets.c b/kernel/arch/arm/kernel/asm-offsets.c
index 70993af22d..6dd7a8f96c 100644
--- a/kernel/arch/arm/kernel/asm-offsets.c
+++ b/kernel/arch/arm/kernel/asm-offsets.c
@@ -42,6 +42,8 @@ int main(void)
 #endif
   BLANK();
   DEFINE(TI_FLAGS,		offsetof(struct thread_info, flags));
+  DEFINE(TI_LOCAL_FLAGS,	offsetof(struct thread_info, local_flags));
+  DEFINE(TI_SYSCALL,		offsetof(struct thread_info, syscall));
   DEFINE(TI_PREEMPT,		offsetof(struct thread_info, preempt_count));
   DEFINE(TI_ADDR_LIMIT,		offsetof(struct thread_info, addr_limit));
   DEFINE(TI_TASK,		offsetof(struct thread_info, task));
@@ -51,6 +53,7 @@ int main(void)
   DEFINE(TI_USED_CP,		offsetof(struct thread_info, used_cp));
   DEFINE(TI_TP_VALUE,		offsetof(struct thread_info, tp_value));
   DEFINE(TI_FPSTATE,		offsetof(struct thread_info, fpstate));
+  DEFINE(TI_OOB_MASK,		STAGE_MASK);
 #ifdef CONFIG_VFP
   DEFINE(TI_VFPSTATE,		offsetof(struct thread_info, vfpstate));
 #ifdef CONFIG_SMP
@@ -161,6 +164,7 @@ int main(void)
   BLANK();
 #ifdef CONFIG_VDSO
   DEFINE(VDSO_DATA_SIZE,	sizeof(union vdso_data_store));
+  DEFINE(VDSO_PRIV_SIZE,	PAGE_SIZE);
 #endif
   BLANK();
 #ifdef CONFIG_ARM_MPU
diff --git a/kernel/arch/arm/kernel/entry-armv.S b/kernel/arch/arm/kernel/entry-armv.S
index 030351d169..a3932c69a0 100644
--- a/kernel/arch/arm/kernel/entry-armv.S
+++ b/kernel/arch/arm/kernel/entry-armv.S
@@ -5,6 +5,7 @@
  *  Copyright (C) 1996,1997,1998 Russell King.
  *  ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk)
  *  nommu support by Hyok S. Choi (hyok.choi@samsung.com)
+ *  Copyright (C) 2005 Stelian Pop.
  *
  *  Low-level vector interface routines
  *
@@ -32,16 +33,24 @@
 #include "entry-header.S"
 #include <asm/entry-macro-multi.S>
 #include <asm/probes.h>
+#include <asm/dovetail.h>
 
 /*
  * Interrupt handling.
  */
 	.macro	irq_handler
 #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
-	ldr	r1, =handle_arch_irq
 	mov	r0, sp
 	badr	lr, 9997f
+#ifdef CONFIG_IRQ_PIPELINE
+	ldr	r1, =handle_arch_irq_pipelined
+	mov	pc, r1
+#else
+	ldr	r1, =handle_arch_irq
 	ldr	pc, [r1]
+#endif
+#elif CONFIG_IRQ_PIPELINE
+#error "Legacy IRQ handling not pipelined"
 #else
 	arch_irq_handler_default
 #endif
@@ -183,7 +192,10 @@ ENDPROC(__und_invalid)
 	uaccess_entry tsk, r0, r1, r2, \uaccess
 
 	.if \trace
-#ifdef CONFIG_TRACE_IRQFLAGS
+#ifdef CONFIG_IRQ_PIPELINE
+	mov	r0, sp
+	bl	kentry_enter_pipelined
+#elif defined(CONFIG_TRACE_IRQFLAGS)
 	bl	trace_hardirqs_off
 #endif
 	.endif
@@ -203,6 +215,10 @@ ENDPROC(__dabt_svc)
 __irq_svc:
 	svc_entry
 	irq_handler
+#ifdef CONFIG_IRQ_PIPELINE
+	tst	r0, r0				@ skip epilogue if oob or in-band stalled
+	beq	1f
+#endif
 
 #ifdef CONFIG_PREEMPTION
 	ldr	r8, [tsk, #TI_PREEMPT]		@ get preempt count
@@ -213,6 +229,7 @@ __irq_svc:
 	blne	svc_preempt
 #endif
 
+1:
 	svc_exit r5, irq = 1			@ return from exception
  UNWIND(.fnend		)
 ENDPROC(__irq_svc)
@@ -222,7 +239,7 @@ ENDPROC(__irq_svc)
 #ifdef CONFIG_PREEMPTION
 svc_preempt:
 	mov	r8, lr
-1:	bl	preempt_schedule_irq		@ irq en/disable is done inside
+1:	bl	arm_preempt_schedule_irq	@ irq en/disable is done inside
 	ldr	r0, [tsk, #TI_FLAGS]		@ get new tasks TI_FLAGS
 	tst	r0, #_TIF_NEED_RESCHED
 	reteq	r8				@ go again
@@ -251,6 +268,16 @@ __und_svc:
 	svc_entry MAX_STACK_SIZE
 #else
 	svc_entry
+#endif
+#ifdef CONFIG_DOVETAIL
+	get_thread_info tsk
+	ldr	r0, [tsk, #TI_PREEMPT]		@ get preempt count
+	tst	r0, #TI_OOB_MASK		@ oob stage?
+	beq	1f
+	mov	r0, #ARM_TRAP_UNDEFINSTR
+	mov	r1, sp				@ r1 = &regs
+	bl	__oob_trap_notify
+1:
 #endif
 
 	mov	r1, #4				@ PC correction to apply
@@ -261,6 +288,15 @@ __und_svc:
 
 __und_svc_finish:
 	get_thread_info tsk
+#ifdef CONFIG_DOVETAIL
+	ldr	r0, [tsk, #TI_PREEMPT]		@ get preempt count
+	tst	r0, #TI_OOB_MASK		@ oob stage?
+	beq	1f
+	mov	r0, #ARM_TRAP_UNDEFINSTR
+	mov	r1, sp				@ r1 = &regs
+	bl	__oob_trap_unwind
+1:
+#endif
 	ldr	r5, [sp, #S_PSR]		@ Get SVC cpsr
 	svc_exit r5				@ return from exception
  UNWIND(.fnend		)
@@ -391,7 +427,7 @@ ENDPROC(__fiq_abt)
 
 	.if	\trace
 #ifdef CONFIG_TRACE_IRQFLAGS
-	bl	trace_hardirqs_off
+	bl	trace_hardirqs_off_pipelined
 #endif
 	ct_user_exit save = 0
 	.endif
@@ -427,8 +463,12 @@ __irq_usr:
 	usr_entry
 	kuser_cmpxchg_check
 	irq_handler
-	get_thread_info tsk
 	mov	why, #0
+#ifdef CONFIG_IRQ_PIPELINE
+	tst	r0, r0
+	beq	fast_ret_to_user	@ skip epilogue if oob (in-band cannot be stalled)
+#endif
+	get_thread_info tsk
 	b	ret_to_user_from_irq
  UNWIND(.fnend		)
 ENDPROC(__irq_usr)
@@ -721,7 +761,7 @@ ENTRY(ret_from_exception)
  UNWIND(.cantunwind	)
 	get_thread_info tsk
 	mov	why, #0
-	b	ret_to_user
+	ret_to_user_pipelined r1
  UNWIND(.fnend		)
 ENDPROC(__pabt_usr)
 ENDPROC(ret_from_exception)
diff --git a/kernel/arch/arm/kernel/entry-common.S b/kernel/arch/arm/kernel/entry-common.S
index 9b3c737575..7a75a49ae6 100644
--- a/kernel/arch/arm/kernel/entry-common.S
+++ b/kernel/arch/arm/kernel/entry-common.S
@@ -3,6 +3,7 @@
  *  linux/arch/arm/kernel/entry-common.S
  *
  *  Copyright (C) 2000 Russell King
+ *  Copyright (C) 2005 Stelian Pop.
  */
 
 #include <asm/assembler.h>
@@ -12,6 +13,7 @@
 #include <asm/memory.h>
 #ifdef CONFIG_AEABI
 #include <asm/unistd-oabi.h>
+#include <uapi/asm-generic/dovetail.h>
 #endif
 
 	.equ	NR_syscalls, __NR_syscalls
@@ -54,6 +56,8 @@ __ret_fast_syscall:
 	blne	addr_limit_check_failed
 	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
 	movs	r1, r1, lsl #16
+	ldr	r2, =#_TIF_SYSCALL_WORK | _TIF_WORK_MASK
+	ands	r2, r1, r2
 	bne	fast_work_pending
 
 
@@ -91,6 +95,8 @@ __ret_fast_syscall:
 	blne	addr_limit_check_failed
 	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
 	movs	r1, r1, lsl #16
+	ldr	r2, =#_TIF_SYSCALL_WORK | _TIF_WORK_MASK
+	ands	r2, r1, r2
 	beq	no_work_pending
  UNWIND(.fnend		)
 ENDPROC(ret_fast_syscall)
@@ -132,6 +138,8 @@ ENTRY(ret_to_user_from_irq)
 	blne	addr_limit_check_failed
 	ldr	r1, [tsk, #TI_FLAGS]
 	movs	r1, r1, lsl #16
+	ldr	r2, =#_TIF_WORK_MASK
+	ands	r2, r1, r2
 	bne	slow_work_pending
 no_work_pending:
 	asm_trace_hardirqs_on save = 0
@@ -143,6 +151,10 @@ no_work_pending:
 	restore_user_regs fast = 0, offset = 0
 ENDPROC(ret_to_user_from_irq)
 ENDPROC(ret_to_user)
+ENTRY(fast_ret_to_user)
+	disable_irq_notrace			@ disable interrupts
+	b	no_work_pending
+ENDPROC(fast_ret_to_user)
 
 /*
  * This is how we return from a fork.
@@ -265,6 +277,9 @@ ENTRY(vector_swi)
 	eor	scno, scno, #__NR_SYSCALL_BASE	@ check OS number
 #endif
 	get_thread_info tsk
+#ifdef CONFIG_DOVETAIL
+	str	scno, [tsk, #TI_SYSCALL]
+#endif
 	/*
 	 * Reload the registers that may have been corrupted on entry to
 	 * the syscall assembly (by tracing or context tracking.)
@@ -272,6 +287,70 @@ ENTRY(vector_swi)
  TRACE(	ldmia	sp, {r0 - r3}		)
 
 local_restart:
+#ifdef CONFIG_DOVETAIL
+	ldr	r10, [tsk, #TI_LOCAL_FLAGS]	@ tsk(r10) is callee-saved
+#ifdef CONFIG_IPIPE_COMPAT
+	ldr	r0, =#0xf0042			@ old syscall signature
+	cmp	scno, r0
+	bne	1f
+	add	scno, scno, #__OOB_SYSCALL_BIT	@ force in oob marker
+	b	fastcall_try
+1:
+#endif	
+#ifdef CONFIG_DOVETAIL_LEGACY_SYSCALL_RANGE
+	ldr	r0, =#__OOB_SYSCALL_BIT
+	ands	r0, scno, r0
+	bne	fastcall_try
+#endif
+	cmp	scno, #__NR_prctl
+	bne	slow_path
+	ldr	r0, [sp, #S_OLD_R0]
+	tst	r0, #__OOB_SYSCALL_BIT
+	beq	slow_path
+fastcall_try:
+	tst	r10, #_TLF_OOB
+	beq	slow_path
+	mov	r0, sp				@ regs
+	bl	handle_oob_syscall
+	ldr	r10, [tsk, #TI_LOCAL_FLAGS]
+	tst	r0, r0
+	beq	slow_path
+	tst	r10, #_TLF_OOB
+	bne	fastcall_exit_check		@ check for MAYDAY
+	bl	sync_inband_irqs
+	b	ret_slow_syscall
+fastcall_exit_check:
+	ldr	r10, [tsk, #TI_FLAGS]
+	tst	r10, #_TIF_MAYDAY
+	beq	fast_ret_to_user
+	mov	r0, sp
+	bl	dovetail_call_mayday
+	b	fast_ret_to_user
+slow_path:
+	tst	r10, #_TLF_DOVETAIL
+	bne	pipeline_syscall
+#ifdef CONFIG_DOVETAIL_LEGACY_SYSCALL_RANGE
+	ldr	r0, =#__OOB_SYSCALL_BIT
+	ands	r0, scno, r0
+	bne	pipeline_syscall
+#endif	
+	cmp	scno, #__NR_prctl
+	bne	root_syscall
+	ldr	r0, [sp, #S_OLD_R0]
+	tst	r0, #__OOB_SYSCALL_BIT
+	beq	root_syscall
+pipeline_syscall:
+	mov	r0, sp				@ regs
+	bl	__pipeline_syscall
+	ldr	r10, [tsk, #TI_LOCAL_FLAGS]
+	tst	r10, #_TLF_OOB
+	bne	fast_ret_to_user
+	cmp	r0, #0
+	bgt	ret_slow_syscall
+root_syscall:
+	ldmia	sp, { r0 - r3 }
+#endif /* CONFIG_DOVETAIL */
+
 	ldr	r10, [tsk, #TI_FLAGS]		@ check for syscall tracing
 	stmdb	sp!, {r4, r5}			@ push fifth and sixth args
 
diff --git a/kernel/arch/arm/kernel/entry-header.S b/kernel/arch/arm/kernel/entry-header.S
index 40db0f9188..da1251c4e6 100644
--- a/kernel/arch/arm/kernel/entry-header.S
+++ b/kernel/arch/arm/kernel/entry-header.S
@@ -203,15 +203,21 @@
 	.macro	svc_exit, rpsr, irq = 0
 	.if	\irq != 0
 	@ IRQs already off
-#ifdef CONFIG_TRACE_IRQFLAGS
 	@ The parent context IRQs must have been enabled to get here in
 	@ the first place, so there's no point checking the PSR I bit.
+#ifdef CONFIG_IRQ_PIPELINE
+	mov	r0, sp
+	bl	kentry_exit_pipelined
+#elif defined(CONFIG_TRACE_IRQFLAGS)
 	bl	trace_hardirqs_on
 #endif
 	.else
 	@ IRQs off again before pulling preserved data off the stack
 	disable_irq_notrace
-#ifdef CONFIG_TRACE_IRQFLAGS
+#ifdef CONFIG_IRQ_PIPELINE
+	mov	r0, sp
+	bl	kentry_exit_pipelined
+#elif defined(CONFIG_TRACE_IRQFLAGS)
 	tst	\rpsr, #PSR_I_BIT
 	bleq	trace_hardirqs_on
 	tst	\rpsr, #PSR_I_BIT
@@ -401,6 +407,19 @@
 #endif
 	.endm
 
+/*
+ * Branch to the exception epilogue, skipping the in-band work
+ * if running over the out-of-band interrupt stage.
+ */
+	.macro ret_to_user_pipelined, tmp
+#ifdef CONFIG_IRQ_PIPELINE
+	ldr	\tmp, [tsk, #TI_LOCAL_FLAGS]
+	tst	\tmp, #_TLF_OOB
+	bne	fast_ret_to_user
+#endif
+	b	ret_to_user
+	.endm
+
 /*
  * These are the registers used in the syscall handler, and allow us to
  * have in theory up to 7 arguments to a function - r0 to r6.
diff --git a/kernel/arch/arm/kernel/irq.c b/kernel/arch/arm/kernel/irq.c
index 698b6f6361..0c3b893a0e 100644
--- a/kernel/arch/arm/kernel/irq.c
+++ b/kernel/arch/arm/kernel/irq.c
@@ -23,6 +23,7 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqchip.h>
+#include <linux/irq_pipeline.h>
 #include <linux/random.h>
 #include <linux/smp.h>
 #include <linux/init.h>
@@ -97,6 +98,14 @@ void __init init_IRQ(void)
 	uniphier_cache_init();
 }
 
+#ifdef CONFIG_IRQ_PIPELINE
+asmlinkage int __exception_irq_entry
+handle_arch_irq_pipelined(struct pt_regs *regs)
+{
+	return handle_irq_pipelined(regs);
+}
+#endif
+
 #ifdef CONFIG_SPARSE_IRQ
 int __init arch_probe_nr_irqs(void)
 {
diff --git a/kernel/arch/arm/kernel/irq_pipeline.c b/kernel/arch/arm/kernel/irq_pipeline.c
new file mode 100644
index 0000000000..aa12dceed8
--- /dev/null
+++ b/kernel/arch/arm/kernel/irq_pipeline.c
@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#include <linux/irq.h>
+#include <linux/irq_pipeline.h>
+
+void arch_do_IRQ_pipelined(struct irq_desc *desc)
+{
+	struct pt_regs *regs = raw_cpu_ptr(&irq_pipeline.tick_regs);
+	unsigned int irq = irq_desc_get_irq(desc);
+
+	__handle_domain_irq(NULL, irq, false, regs);
+}
+
+void __init arch_irq_pipeline_init(void)
+{
+	/* no per-arch init. */
+}
diff --git a/kernel/arch/arm/kernel/patch.c b/kernel/arch/arm/kernel/patch.c
index e9e828b6bb..35c7285bfd 100644
--- a/kernel/arch/arm/kernel/patch.c
+++ b/kernel/arch/arm/kernel/patch.c
@@ -17,7 +17,7 @@ struct patch {
 };
 
 #ifdef CONFIG_MMU
-static DEFINE_RAW_SPINLOCK(patch_lock);
+static DEFINE_HARD_SPINLOCK(patch_lock);
 
 static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags)
 {
diff --git a/kernel/arch/arm/kernel/process.c b/kernel/arch/arm/kernel/process.c
index 47a30ff888..d8da49a7dc 100644
--- a/kernel/arch/arm/kernel/process.c
+++ b/kernel/arch/arm/kernel/process.c
@@ -71,6 +71,7 @@ void arch_cpu_idle(void)
 		arm_pm_idle();
 	else
 		cpu_do_idle();
+	hard_cond_local_irq_enable();
 	raw_local_irq_enable();
 }
 
@@ -448,3 +449,28 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 	return ret;
 }
 #endif
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+/*
+ * When pipelining interrupts, we have to reconcile the hardware and
+ * the virtual states. Hard irqs are off on entry while the current
+ * stage has to be unstalled: fix this up by stalling the in-band
+ * stage on entry, unstalling on exit.
+ */
+asmlinkage void __sched arm_preempt_schedule_irq(void)
+{
+	WARN_ON_ONCE(irq_pipeline_debug() && test_inband_stall());
+	stall_inband_nocheck();
+	preempt_schedule_irq();
+	unstall_inband_nocheck();
+}
+
+#else
+
+asmlinkage void __sched arm_preempt_schedule_irq(void)
+{
+	preempt_schedule_irq();
+}
+
+#endif
diff --git a/kernel/arch/arm/kernel/ptrace.c b/kernel/arch/arm/kernel/ptrace.c
index 2771e68222..1074670a47 100644
--- a/kernel/arch/arm/kernel/ptrace.c
+++ b/kernel/arch/arm/kernel/ptrace.c
@@ -205,7 +205,9 @@ void ptrace_break(struct pt_regs *regs)
 
 static int break_trap(struct pt_regs *regs, unsigned int instr)
 {
+	mark_trap_entry(ARM_TRAP_BREAK, regs);
 	ptrace_break(regs);
+	mark_trap_exit(ARM_TRAP_BREAK, regs);
 	return 0;
 }
 
diff --git a/kernel/arch/arm/kernel/raw_printk.c b/kernel/arch/arm/kernel/raw_printk.c
new file mode 100644
index 0000000000..9024b772fc
--- /dev/null
+++ b/kernel/arch/arm/kernel/raw_printk.c
@@ -0,0 +1,30 @@
+#include <linux/kernel.h>
+#include <linux/console.h>
+#include <linux/init.h>
+
+/*
+ * If both CONFIG_DEBUG_LL and CONFIG_RAW_PRINTK are set, create a
+ * console device sending the raw output to printascii().
+ */
+void printascii(const char *s);
+
+static void raw_console_write(struct console *co,
+			      const char *s, unsigned count)
+{
+	printascii(s);
+}
+
+static struct console raw_console = {
+	.name		= "rawcon",
+	.write_raw	= raw_console_write,
+	.flags		= CON_PRINTBUFFER | CON_ENABLED,
+	.index		= -1,
+};
+
+static int __init raw_console_init(void)
+{
+	register_console(&raw_console);
+
+	return 0;
+}
+console_initcall(raw_console_init);
diff --git a/kernel/arch/arm/kernel/signal.c b/kernel/arch/arm/kernel/signal.c
index a3a38d0a4c..da78d2fbdd 100644
--- a/kernel/arch/arm/kernel/signal.c
+++ b/kernel/arch/arm/kernel/signal.c
@@ -8,6 +8,7 @@
 #include <linux/random.h>
 #include <linux/signal.h>
 #include <linux/personality.h>
+#include <linux/irq_pipeline.h>
 #include <linux/uaccess.h>
 #include <linux/tracehook.h>
 #include <linux/uprobes.h>
@@ -639,16 +640,36 @@ static int do_signal(struct pt_regs *regs, int syscall)
 	return 0;
 }
 
+static inline void do_retuser(void)
+{
+	unsigned int thread_flags;
+
+	if (dovetailing()) {
+		thread_flags = current_thread_info()->flags;
+		if (thread_flags & _TIF_RETUSER)
+			inband_retuser_notify();
+	}
+}
+
 asmlinkage int
 do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall)
 {
+	WARN_ON_ONCE(irq_pipeline_debug() &&
+		(irqs_disabled() || running_oob()));
+
 	/*
 	 * The assembly code enters us with IRQs off, but it hasn't
 	 * informed the tracing code of that for efficiency reasons.
 	 * Update the trace code with the current status.
 	 */
-	trace_hardirqs_off();
+	if (!irqs_pipelined())
+		trace_hardirqs_off();
 	do {
+		if (irqs_pipelined()) {
+			local_irq_disable();
+			hard_cond_local_irq_enable();
+		}
+
 		if (likely(thread_flags & _TIF_NEED_RESCHED)) {
 			schedule();
 		} else {
@@ -658,6 +679,7 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall)
 			if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) {
 				int restart = do_signal(regs, syscall);
 				if (unlikely(restart)) {
+					do_retuser();
 					/*
 					 * Restart without handlers.
 					 * Deal with it without leaving
@@ -672,10 +694,16 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall)
 				tracehook_notify_resume(regs);
 				rseq_handle_notify_resume(NULL, regs);
 			}
+			do_retuser();
 		}
-		local_irq_disable();
+		hard_local_irq_disable();
+
+		/* RETUSER might have switched oob */
+		if (!running_inband())
+			break;
+
 		thread_flags = current_thread_info()->flags;
-	} while (thread_flags & _TIF_WORK_MASK);
+	} while (inband_irq_pending() || (thread_flags & _TIF_WORK_MASK));
 	return 0;
 }
 
diff --git a/kernel/arch/arm/kernel/smp.c b/kernel/arch/arm/kernel/smp.c
index 123432bb50..bdb4f7e978 100644
--- a/kernel/arch/arm/kernel/smp.c
+++ b/kernel/arch/arm/kernel/smp.c
@@ -84,7 +84,7 @@ enum ipi_msg_type {
 	MAX_IPI
 };
 
-static int ipi_irq_base __read_mostly;
+int ipi_irq_base __read_mostly;
 static int nr_ipi __read_mostly = NR_IPI;
 static struct irq_desc *ipi_desc[MAX_IPI] __read_mostly;
 
@@ -329,7 +329,7 @@ void arch_cpu_idle_dead(void)
 
 	idle_task_exit();
 
-	local_irq_disable();
+	local_irq_disable_full();
 
 	/*
 	 * Flush the data out of the L1 cache for this CPU.  This must be
@@ -420,6 +420,13 @@ asmlinkage void secondary_start_kernel(void)
 	enter_lazy_tlb(mm, current);
 	local_flush_tlb_all();
 
+	/*
+	 * irq_pipeline: debug_smp_processor_id() accesses percpu
+	 * data.
+	 */
+	if (irqs_pipelined())
+		set_my_cpu_offset(per_cpu_offset(raw_smp_processor_id()));
+
 	/*
 	 * All kernel threads share the same mm context; grab a
 	 * reference and switch to it.
@@ -463,7 +470,7 @@ asmlinkage void secondary_start_kernel(void)
 
 	complete(&cpu_running);
 
-	local_irq_enable();
+	local_irq_enable_full();
 	local_fiq_enable();
 	local_abt_enable();
 
@@ -539,6 +546,8 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
 
 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);
 
+static unsigned int get_ipi_count(unsigned int irq, unsigned int cpu);
+
 void show_ipi_list(struct seq_file *p, int prec)
 {
 	unsigned int cpu, i;
@@ -553,7 +562,7 @@ void show_ipi_list(struct seq_file *p, int prec)
 		seq_printf(p, "%*s%u: ", prec - 1, "IPI", i);
 
 		for_each_online_cpu(cpu)
-			seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu));
+			seq_printf(p, "%10u ", get_ipi_count(irq, cpu));
 
 		seq_printf(p, " %s\n", ipi_types[i]);
 	}
@@ -606,7 +615,7 @@ static void ipi_cpu_stop(unsigned int cpu)
 	set_cpu_online(cpu, false);
 
 	local_fiq_disable();
-	local_irq_disable();
+	local_irq_disable_full();
 
 	while (1) {
 		cpu_relax();
@@ -695,6 +704,12 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
+	/*
+	 * We don't support legacy IPI delivery when pipelining
+	 * interrupts.
+	 */
+	WARN_ON_ONCE(irqs_pipelined());
+
 	irq_enter();
 	do_handle_IPI(ipinr);
 	irq_exit();
@@ -702,6 +717,73 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
 	set_irq_regs(old_regs);
 }
 
+static void __smp_cross_call(const struct cpumask *target, unsigned int ipinr)
+{
+	trace_ipi_raise(target, ipi_types[ipinr]);
+	__ipi_send_mask(ipi_desc[ipinr], target);
+}
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+static DEFINE_PER_CPU(unsigned long, ipi_messages);
+
+static DEFINE_PER_CPU(unsigned int [MAX_IPI], ipi_counts);
+
+static irqreturn_t ipi_handler(int irq, void *data)
+{
+	unsigned long *pmsg;
+	unsigned int ipinr;
+
+	/*
+	 * Decode in-band IPIs (0..MAX_IPI - 1) multiplexed over
+	 * SGI0. Out-of-band IPIs (SGI1, SGI2) have their own
+	 * individual handler.
+	 */
+	pmsg = raw_cpu_ptr(&ipi_messages);
+	while (*pmsg) {
+		ipinr = ffs(*pmsg) - 1;
+		clear_bit(ipinr, pmsg);
+		__this_cpu_inc(ipi_counts[ipinr]);
+		do_handle_IPI(ipinr);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
+{
+	unsigned int cpu;
+
+	/* regular in-band IPI (multiplexed over SGI0). */
+	for_each_cpu(cpu, target)
+		set_bit(ipinr, &per_cpu(ipi_messages, cpu));
+
+	wmb();
+	__smp_cross_call(target, 0);
+}
+
+static unsigned int get_ipi_count(unsigned int irq, unsigned int cpu)
+{
+	return per_cpu(ipi_counts[irq - ipi_irq_base], cpu);
+}
+
+void irq_send_oob_ipi(unsigned int irq,
+		const struct cpumask *cpumask)
+{
+	unsigned int sgi = irq - ipi_irq_base;
+
+	if (WARN_ON(irq_pipeline_debug() &&
+		    (sgi < OOB_IPI_OFFSET ||
+		     sgi >= OOB_IPI_OFFSET + OOB_NR_IPI)))
+		return;
+
+	/* Out-of-band IPI (SGI1-2). */
+	__smp_cross_call(cpumask, sgi);
+}
+EXPORT_SYMBOL_GPL(irq_send_oob_ipi);
+
+#else
+
 static irqreturn_t ipi_handler(int irq, void *data)
 {
 	do_handle_IPI(irq - ipi_irq_base);
@@ -710,10 +792,16 @@ static irqreturn_t ipi_handler(int irq, void *data)
 
 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
 {
-	trace_ipi_raise_rcuidle(target, ipi_types[ipinr]);
-	__ipi_send_mask(ipi_desc[ipinr], target);
+	__smp_cross_call(target, ipinr);
+}
+
+static unsigned int get_ipi_count(unsigned int irq, unsigned int cpu)
+{
+	return kstat_irqs_cpu(irq, cpu);
 }
 
+#endif /* CONFIG_IRQ_PIPELINE */
+
 static void ipi_setup(int cpu)
 {
 	int i;
@@ -727,18 +815,25 @@ static void ipi_setup(int cpu)
 
 void __init set_smp_ipi_range(int ipi_base, int n)
 {
-	int i;
+	int i, inband_nr_ipi;
 
 	WARN_ON(n < MAX_IPI);
 	nr_ipi = min(n, MAX_IPI);
+	/*
+	 * irq_pipeline: the in-band stage traps SGI0 only,
+	 * over which IPI messages are mutiplexed. Other SGIs
+	 * are available for exchanging out-of-band IPIs.
+	 */
+	inband_nr_ipi = irqs_pipelined() ? 1 : nr_ipi;
 
 	for (i = 0; i < nr_ipi; i++) {
-		int err;
-
-		err = request_percpu_irq(ipi_base + i, ipi_handler,
-					 "IPI", &irq_stat);
-		WARN_ON(err);
+		if (i < inband_nr_ipi) {
+			int err;
 
+			err = request_percpu_irq(ipi_base + i, ipi_handler,
+						"IPI", &irq_stat);
+			WARN_ON(err);
+		}
 		ipi_desc[i] = irq_to_desc(ipi_base + i);
 		irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
 
diff --git a/kernel/arch/arm/kernel/smp_twd.c b/kernel/arch/arm/kernel/smp_twd.c
index 9a14f721a2..8377f1d030 100644
--- a/kernel/arch/arm/kernel/smp_twd.c
+++ b/kernel/arch/arm/kernel/smp_twd.c
@@ -31,7 +31,7 @@ static DEFINE_PER_CPU(bool, percpu_setup_called);
 
 static struct clock_event_device __percpu *twd_evt;
 static unsigned int twd_features =
-		CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT;
+		CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PIPELINE;
 static int twd_ppi;
 
 static int twd_shutdown(struct clock_event_device *clk)
@@ -182,7 +182,7 @@ static irqreturn_t twd_handler(int irq, void *dev_id)
 	struct clock_event_device *evt = dev_id;
 
 	if (twd_timer_ack()) {
-		evt->event_handler(evt);
+		clockevents_handle_event(evt);
 		return IRQ_HANDLED;
 	}
 
@@ -279,7 +279,8 @@ static int __init twd_local_timer_common_register(struct device_node *np)
 		goto out_free;
 	}
 
-	err = request_percpu_irq(twd_ppi, twd_handler, "twd", twd_evt);
+	err = __request_percpu_irq(twd_ppi, twd_handler,
+				   IRQF_TIMER, "twd", twd_evt);
 	if (err) {
 		pr_err("twd: can't register interrupt %d (%d)\n", twd_ppi, err);
 		goto out_free;
diff --git a/kernel/arch/arm/kernel/traps.c b/kernel/arch/arm/kernel/traps.c
index a531afad87..e33ea9c0d7 100644
--- a/kernel/arch/arm/kernel/traps.c
+++ b/kernel/arch/arm/kernel/traps.c
@@ -406,7 +406,7 @@ int is_valid_bugaddr(unsigned long pc)
 #endif
 
 static LIST_HEAD(undef_hook);
-static DEFINE_RAW_SPINLOCK(undef_lock);
+static DEFINE_HARD_SPINLOCK(undef_lock);
 
 void register_undef_hook(struct undef_hook *hook)
 {
diff --git a/kernel/arch/arm/kernel/vdso.c b/kernel/arch/arm/kernel/vdso.c
index fddd08a6e0..557fb35f69 100644
--- a/kernel/arch/arm/kernel/vdso.c
+++ b/kernel/arch/arm/kernel/vdso.c
@@ -32,7 +32,10 @@ static struct page **vdso_text_pagelist;
 
 extern char vdso_start[], vdso_end[];
 
-/* Total number of pages needed for the data and text portions of the VDSO. */
+/*
+ * Total number of pages needed for the data, private and text
+ * portions of the VDSO.
+ */
 unsigned int vdso_total_pages __ro_after_init;
 
 /*
@@ -53,8 +56,8 @@ static int vdso_mremap(const struct vm_special_mapping *sm,
 	unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
 	unsigned long vdso_size;
 
-	/* without VVAR page */
-	vdso_size = (vdso_total_pages - 1) << PAGE_SHIFT;
+	/* without VVAR and VPRIV pages */
+	vdso_size = (vdso_total_pages - 2) << PAGE_SHIFT;
 
 	if (vdso_size != new_size)
 		return -EINVAL;
@@ -180,8 +183,10 @@ static void __init patch_vdso(void *ehdr)
 	/* If the virtual counter is absent or non-functional we don't
 	 * want programs to incur the slight additional overhead of
 	 * dispatching through the VDSO only to fall back to syscalls.
+	 * However, if clocksources supporting generic MMIO access can
+	 * be reached via the vDSO, keep this fast path enabled.
 	 */
-	if (!cntvct_ok) {
+	if (!cntvct_ok && !IS_ENABLED(CONFIG_GENERIC_CLOCKSOURCE_VDSO)) {
 		vdso_nullpatch_one(&einfo, "__vdso_gettimeofday");
 		vdso_nullpatch_one(&einfo, "__vdso_clock_gettime");
 		vdso_nullpatch_one(&einfo, "__vdso_clock_gettime64");
@@ -219,17 +224,27 @@ static int __init vdso_init(void)
 
 	vdso_text_mapping.pages = vdso_text_pagelist;
 
-	vdso_total_pages = 1; /* for the data/vvar page */
+	vdso_total_pages = 2; /* for the data/vvar and vpriv pages */
 	vdso_total_pages += text_pages;
 
 	cntvct_ok = cntvct_functional();
 
 	patch_vdso(vdso_start);
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+	vdso_data->cs_type_seq = CLOCKSOURCE_VDSO_NONE << 16 | 1;
+#endif
 
 	return 0;
 }
 arch_initcall(vdso_init);
 
+static int install_vpriv(struct mm_struct *mm, unsigned long addr)
+{
+	return mmap_region(NULL, addr, PAGE_SIZE,
+			  VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE,
+			   0, NULL) != addr ? -EINVAL : 0;
+}
+
 static int install_vvar(struct mm_struct *mm, unsigned long addr)
 {
 	struct vm_area_struct *vma;
@@ -237,8 +252,13 @@ static int install_vvar(struct mm_struct *mm, unsigned long addr)
 	vma = _install_special_mapping(mm, addr, PAGE_SIZE,
 				       VM_READ | VM_MAYREAD,
 				       &vdso_data_mapping);
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
+
+	if (cache_is_vivt())
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
-	return PTR_ERR_OR_ZERO(vma);
+	return vma->vm_start != addr ? -EINVAL : 0;
 }
 
 /* assumes mmap_lock is write-locked */
@@ -252,18 +272,29 @@ void arm_install_vdso(struct mm_struct *mm, unsigned long addr)
 	if (vdso_text_pagelist == NULL)
 		return;
 
-	if (install_vvar(mm, addr))
+	if (install_vpriv(mm, addr)) {
+		pr_err("cannot map VPRIV at expected address!\n");
 		return;
+	}
+
+	/* Account for the private storage. */
+	addr += PAGE_SIZE;
+	if (install_vvar(mm, addr)) {
+		WARN(1, "cannot map VVAR at expected address!\n");
+		return;
+	}
 
-	/* Account for vvar page. */
+	/* Account for vvar and vpriv pages. */
 	addr += PAGE_SIZE;
-	len = (vdso_total_pages - 1) << PAGE_SHIFT;
+	len = (vdso_total_pages - 2) << PAGE_SHIFT;
 
 	vma = _install_special_mapping(mm, addr, len,
 		VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC,
 		&vdso_text_mapping);
 
-	if (!IS_ERR(vma))
+	if (IS_ERR(vma) || vma->vm_start != addr)
+		WARN(1, "cannot map VDSO at expected address!\n");
+	else
 		mm->context.vdso = addr;
 }
 
diff --git a/kernel/arch/arm/mach-imx/gpc.c b/kernel/arch/arm/mach-imx/gpc.c
index ebc4339b8b..189642e4f2 100644
--- a/kernel/arch/arm/mach-imx/gpc.c
+++ b/kernel/arch/arm/mach-imx/gpc.c
@@ -62,28 +62,38 @@ void imx_gpc_set_l2_mem_power_in_lpm(bool power_off)
 void imx_gpc_pre_suspend(bool arm_power_off)
 {
 	void __iomem *reg_imr1 = gpc_base + GPC_IMR1;
+	unsigned long flags;
 	int i;
 
 	/* Tell GPC to power off ARM core when suspend */
 	if (arm_power_off)
 		imx_gpc_set_arm_power_in_lpm(arm_power_off);
 
+	flags = hard_cond_local_irq_save();
+
 	for (i = 0; i < IMR_NUM; i++) {
 		gpc_saved_imrs[i] = readl_relaxed(reg_imr1 + i * 4);
 		writel_relaxed(~gpc_wake_irqs[i], reg_imr1 + i * 4);
 	}
+
+	hard_cond_local_irq_restore(flags);
 }
 
 void imx_gpc_post_resume(void)
 {
 	void __iomem *reg_imr1 = gpc_base + GPC_IMR1;
+	unsigned long flags;
 	int i;
 
 	/* Keep ARM core powered on for other low-power modes */
 	imx_gpc_set_arm_power_in_lpm(false);
 
+	flags = hard_cond_local_irq_save();
+
 	for (i = 0; i < IMR_NUM; i++)
 		writel_relaxed(gpc_saved_imrs[i], reg_imr1 + i * 4);
+
+	hard_cond_local_irq_restore(flags);
 }
 
 static int imx_gpc_irq_set_wake(struct irq_data *d, unsigned int on)
@@ -105,21 +115,31 @@ static int imx_gpc_irq_set_wake(struct irq_data *d, unsigned int on)
 void imx_gpc_mask_all(void)
 {
 	void __iomem *reg_imr1 = gpc_base + GPC_IMR1;
+	unsigned long flags;
 	int i;
 
+	flags = hard_cond_local_irq_save();
+
 	for (i = 0; i < IMR_NUM; i++) {
 		gpc_saved_imrs[i] = readl_relaxed(reg_imr1 + i * 4);
 		writel_relaxed(~0, reg_imr1 + i * 4);
 	}
+
+	hard_cond_local_irq_restore(flags);
 }
 
 void imx_gpc_restore_all(void)
 {
 	void __iomem *reg_imr1 = gpc_base + GPC_IMR1;
+	unsigned long flags;
 	int i;
 
+	flags = hard_cond_local_irq_save();
+
 	for (i = 0; i < IMR_NUM; i++)
 		writel_relaxed(gpc_saved_imrs[i], reg_imr1 + i * 4);
+
+	hard_cond_local_irq_restore(flags);
 }
 
 void imx_gpc_hwirq_unmask(unsigned int hwirq)
@@ -167,6 +187,7 @@ static struct irq_chip imx_gpc_chip = {
 #ifdef CONFIG_SMP
 	.irq_set_affinity	= irq_chip_set_affinity_parent,
 #endif
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static int imx_gpc_domain_translate(struct irq_domain *d,
diff --git a/kernel/arch/arm/mm/alignment.c b/kernel/arch/arm/mm/alignment.c
index bcefe3f517..6018226328 100644
--- a/kernel/arch/arm/mm/alignment.c
+++ b/kernel/arch/arm/mm/alignment.c
@@ -19,6 +19,7 @@
 #include <linux/init.h>
 #include <linux/sched/signal.h>
 #include <linux/uaccess.h>
+#include <linux/dovetail.h>
 
 #include <asm/cp15.h>
 #include <asm/system_info.h>
@@ -807,10 +808,12 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	u16 tinstr = 0;
 	int isize = 4;
 	int thumb2_32b = 0;
-	int fault;
+	int fault, ret = 0;
 
 	if (interrupts_enabled(regs))
-		local_irq_enable();
+		hard_local_irq_enable();
+
+	mark_trap_entry(ARM_TRAP_ALIGNMENT, regs);
 
 	instrptr = instruction_pointer(regs);
 
@@ -938,7 +941,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	if (thumb_mode(regs))
 		regs->ARM_cpsr = it_advance(regs->ARM_cpsr);
 
-	return 0;
+	goto out;
 
  bad_or_fault:
 	if (type == TYPE_ERROR)
@@ -947,7 +950,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	 * We got a fault - fix it up, or die.
 	 */
 	do_bad_area(addr, fsr, regs);
-	return 0;
+	goto out;
 
  swp:
 	pr_err("Alignment trap: not handling swp instruction\n");
@@ -961,7 +964,8 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 		isize << 1,
 		isize == 2 ? tinstr : instr, instrptr);
 	ai_skipped += 1;
-	return 1;
+	ret = 1;
+	goto out;
 
  user:
 	ai_user += 1;
@@ -992,12 +996,15 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 		 * entry-common.S) and disable the alignment trap only if
 		 * there is no work pending for this thread.
 		 */
-		raw_local_irq_disable();
+		hard_local_irq_disable();
 		if (!(current_thread_info()->flags & _TIF_WORK_MASK))
 			set_cr(cr_no_alignment);
 	}
 
-	return 0;
+out:
+	mark_trap_exit(ARM_TRAP_ALIGNMENT, regs);
+
+	return ret;
 }
 
 static int __init noalign_setup(char *__unused)
diff --git a/kernel/arch/arm/mm/cache-l2x0.c b/kernel/arch/arm/mm/cache-l2x0.c
index 43d91bfd23..b2af3e0da8 100644
--- a/kernel/arch/arm/mm/cache-l2x0.c
+++ b/kernel/arch/arm/mm/cache-l2x0.c
@@ -38,7 +38,7 @@ struct l2c_init_data {
 
 static void __iomem *l2x0_base;
 static const struct l2c_init_data *l2x0_data;
-static DEFINE_RAW_SPINLOCK(l2x0_lock);
+static DEFINE_HARD_SPINLOCK(l2x0_lock);
 static u32 l2x0_way_mask;	/* Bitmask of active ways */
 static u32 l2x0_size;
 static unsigned long sync_reg_offset = L2X0_CACHE_SYNC;
@@ -48,6 +48,19 @@ struct l2x0_regs l2x0_saved_regs;
 static bool l2x0_bresp_disable;
 static bool l2x0_flz_disable;
 
+#ifdef CONFIG_IRQ_PIPELINE
+#define CACHE_RANGE_ATOMIC_MAX	512UL
+static int l2x0_wa = -1;
+static int __init l2x0_setup_wa(char *str)
+{
+	l2x0_wa = !!simple_strtol(str, NULL, 0);
+	return 0;
+}
+early_param("l2x0_write_allocate", l2x0_setup_wa);
+#else
+#define CACHE_RANGE_ATOMIC_MAX	4096UL
+#endif
+
 /*
  * Common code for all cache controllers.
  */
@@ -120,11 +133,11 @@ static void l2c_enable(void __iomem *base, unsigned num_lock)
 
 	l2x0_data->unlock(base, num_lock);
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	__l2c_op_way(base + L2X0_INV_WAY);
 	writel_relaxed(0, base + sync_reg_offset);
 	l2c_wait_mask(base + sync_reg_offset, 1);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	l2c_write_sec(L2X0_CTRL_EN, base, L2X0_CTRL);
 }
@@ -225,7 +238,7 @@ static void l2c210_flush_all(void)
 {
 	void __iomem *base = l2x0_base;
 
-	BUG_ON(!irqs_disabled());
+	BUG_ON(!hard_irqs_disabled());
 
 	__l2c_op_way(base + L2X0_CLEAN_INV_WAY);
 	__l2c210_cache_sync(base);
@@ -284,10 +297,10 @@ static void l2c220_op_way(void __iomem *base, unsigned reg)
 static unsigned long l2c220_op_pa_range(void __iomem *reg, unsigned long start,
 	unsigned long end, unsigned long flags)
 {
-	raw_spinlock_t *lock = &l2x0_lock;
+	typeof(l2x0_lock) *lock = &l2x0_lock;
 
 	while (start < end) {
-		unsigned long blk_end = start + min(end - start, 4096UL);
+		unsigned long blk_end = start + min(end - start, CACHE_RANGE_ATOMIC_MAX);
 
 		while (start < blk_end) {
 			l2c_wait_mask(reg, 1);
@@ -498,13 +511,13 @@ static void l2c310_inv_range_erratum(unsigned long start, unsigned long end)
 
 static void l2c310_flush_range_erratum(unsigned long start, unsigned long end)
 {
-	raw_spinlock_t *lock = &l2x0_lock;
+	typeof(l2x0_lock) *lock = &l2x0_lock;
 	unsigned long flags;
 	void __iomem *base = l2x0_base;
 
 	raw_spin_lock_irqsave(lock, flags);
 	while (start < end) {
-		unsigned long blk_end = start + min(end - start, 4096UL);
+		unsigned long blk_end = start + min(end - start, CACHE_RANGE_ATOMIC_MAX);
 
 		l2c_set_debug(base, 0x03);
 		while (start < blk_end) {
@@ -800,6 +813,24 @@ static int __init __l2c_init(const struct l2c_init_data *data,
 	if (aux_val & aux_mask)
 		pr_alert("L2C: platform provided aux values permit register corruption.\n");
 
+#ifdef CONFIG_IRQ_PIPELINE
+	if (!l2x0_wa) {
+		/*
+		 * Disable WA by setting bit 23 in the auxiliary
+		 * control register.
+		 */
+		aux_mask &= ~L220_AUX_CTRL_FWA_MASK;
+		aux_val &= ~L220_AUX_CTRL_FWA_MASK;
+		aux_val |= 1 << L220_AUX_CTRL_FWA_SHIFT;
+		pr_warn("%s: irq_pipeline: write-allocate disabled via command line\n",
+			data->type);
+	} else if ((cache_id & L2X0_CACHE_ID_PART_MASK) == L2X0_CACHE_ID_PART_L220 ||
+		   ((cache_id & L2X0_CACHE_ID_PART_MASK) == L2X0_CACHE_ID_PART_L310 &&
+		    (cache_id & L2X0_CACHE_ID_RTL_MASK) < L310_CACHE_ID_RTL_R3P2))
+		pr_alert("%s: irq_pipeline: write-allocate enabled, may induce high latency\n",
+			 data->type);
+#endif
+
 	old_aux = aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL);
 	aux &= aux_mask;
 	aux |= aux_val;
diff --git a/kernel/arch/arm/mm/context.c b/kernel/arch/arm/mm/context.c
index b7525b433f..0cf14bd0c2 100644
--- a/kernel/arch/arm/mm/context.c
+++ b/kernel/arch/arm/mm/context.c
@@ -39,7 +39,7 @@
 #define ASID_FIRST_VERSION	(1ULL << ASID_BITS)
 #define NUM_USER_ASIDS		ASID_FIRST_VERSION
 
-static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
+static DEFINE_HARD_SPINLOCK(cpu_asid_lock);
 static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
 static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
 
@@ -237,9 +237,12 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu)
 void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
 {
 	unsigned long flags;
-	unsigned int cpu = smp_processor_id();
+	unsigned int cpu = raw_smp_processor_id();
+	bool need_flush;
 	u64 asid;
 
+	WARN_ON_ONCE(dovetail_debug() && !hard_irqs_disabled());
+
 	if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
 		__check_vmalloc_seq(mm);
 
@@ -263,15 +266,16 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
 		atomic64_set(&mm->context.id, asid);
 	}
 
-	if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
-		local_flush_bp_all();
-		local_flush_tlb_all();
-	}
-
+	need_flush = cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending);
 	atomic64_set(&per_cpu(active_asids, cpu), asid);
 	cpumask_set_cpu(cpu, mm_cpumask(mm));
 	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
 
+	if (need_flush) {
+		local_flush_bp_all();
+		local_flush_tlb_all();
+	}
+
 switch_mm_fastpath:
 	cpu_switch_mm(mm->pgd, mm);
 }
diff --git a/kernel/arch/arm/mm/fault.c b/kernel/arch/arm/mm/fault.c
index efa4020250..e23d0ff503 100644
--- a/kernel/arch/arm/mm/fault.c
+++ b/kernel/arch/arm/mm/fault.c
@@ -9,6 +9,7 @@
 #include <linux/signal.h>
 #include <linux/mm.h>
 #include <linux/hardirq.h>
+#include <linux/irq_pipeline.h>
 #include <linux/init.h>
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
@@ -21,11 +22,69 @@
 #include <asm/system_misc.h>
 #include <asm/system_info.h>
 #include <asm/tlbflush.h>
+#include <asm/dovetail.h>
+#define CREATE_TRACE_POINTS
+#include <asm/trace/exceptions.h>
 
 #include "fault.h"
 
 #ifdef CONFIG_MMU
 
+#ifdef CONFIG_IRQ_PIPELINE
+/*
+ * We need to synchronize the virtual interrupt state with the hard
+ * interrupt state we received on entry, then turn hardirqs back on to
+ * allow code which does not require strict serialization to be
+ * preempted by an out-of-band activity.
+ */
+static inline
+unsigned long fault_entry(int exception, struct pt_regs *regs)
+{
+	unsigned long flags;
+
+	trace_ARM_trap_entry(exception, regs);
+
+	flags = hard_local_save_flags();
+
+	/*
+	 * The companion core must demote the current context to
+	 * in-band stage if running oob on entry.
+	 */
+	mark_trap_entry(exception, regs);
+
+	if (raw_irqs_disabled_flags(flags)) {
+		stall_inband();
+		trace_hardirqs_off();
+	}
+
+	hard_local_irq_enable();
+
+	return flags;
+}
+
+static inline
+void fault_exit(int exception, struct pt_regs *regs,
+		unsigned long flags)
+{
+	WARN_ON_ONCE(irq_pipeline_debug() && hard_irqs_disabled());
+
+	/*
+	 * We expect kentry_exit_pipelined() to clear the stall bit if
+	 * kentry_enter_pipelined() observed it that way.
+	 */
+	mark_trap_exit(exception, regs);
+	trace_ARM_trap_exit(exception, regs);
+	hard_local_irq_restore(flags);
+}
+
+#else	/* !CONFIG_IRQ_PIPELINE */
+
+#define fault_entry(__exception, __regs)  ({ 0; })
+#define fault_exit(__exception, __regs, __flags)  \
+	do { (void)(__flags); } while (0)
+
+#endif	/* !CONFIG_IRQ_PIPELINE */
+
 /*
  * This is useful to dump out the page tables associated with
  * 'addr' in mm 'mm'.
@@ -96,6 +155,15 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
 	pr_cont("\n");
 }
 #else					/* CONFIG_MMU */
+unsigned long fault_entry(int exception, struct pt_regs *regs)
+{
+	return 0;
+}
+
+static inline void fault_exit(int exception, struct pt_regs *regs,
+			unsigned long combo)
+{ }
+
 void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
 { }
 #endif					/* CONFIG_MMU */
@@ -116,6 +184,7 @@ __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
 	/*
 	 * No handler, we'll have to terminate things with extreme prejudice.
 	 */
+	irq_pipeline_oops();
 	bust_spinlocks(1);
 	pr_alert("8<--- cut here ---\n");
 	pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
@@ -168,14 +237,22 @@ void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
 	struct task_struct *tsk = current;
 	struct mm_struct *mm = tsk->active_mm;
+	unsigned long irqflags;
 
 	/*
 	 * If we are in kernel mode at this point, we
 	 * have no context to handle this fault with.
 	 */
-	if (user_mode(regs))
+	  if (user_mode(regs)) {
+		irqflags = fault_entry(ARM_TRAP_ACCESS, regs);
 		__do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
-	else
+		fault_exit(ARM_TRAP_ACCESS, regs, irqflags);
+	  } else
+		/*
+		 * irq_pipeline: kernel faults are either quickly
+		 * recoverable via fixup, or lethal. In both cases, we
+		 * can skip the interrupt state synchronization.
+		 */
 		__do_kernel_fault(mm, addr, fsr, regs);
 }
 
@@ -244,9 +321,12 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	int sig, code;
 	vm_fault_t fault;
 	unsigned int flags = FAULT_FLAG_DEFAULT;
+	unsigned long irqflags;
+
+	irqflags = fault_entry(ARM_TRAP_ACCESS, regs);
 
 	if (kprobe_page_fault(regs, fsr))
-		return 0;
+		goto out;
 
 	tsk = current;
 	mm  = tsk->mm;
@@ -302,7 +382,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	if (fault_signal_pending(fault, regs)) {
 		if (!user_mode(regs))
 			goto no_context;
-		return 0;
+		goto out;
 	}
 
 	if (!(fault & VM_FAULT_ERROR) && flags & FAULT_FLAG_ALLOW_RETRY) {
@@ -318,7 +398,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	 * Handle the "normal" case first - VM_FAULT_MAJOR
 	 */
 	if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS))))
-		return 0;
+		goto out;
 
 	/*
 	 * If we are in kernel mode at this point, we
@@ -334,7 +414,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 		 * got oom-killed)
 		 */
 		pagefault_out_of_memory();
-		return 0;
+		goto out;
 	}
 
 	if (fault & VM_FAULT_SIGBUS) {
@@ -355,10 +435,13 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 	}
 
 	__do_user_fault(addr, fsr, sig, code, regs);
-	return 0;
+	goto out;
 
 no_context:
 	__do_kernel_fault(mm, addr, fsr, regs);
+out:
+	fault_exit(ARM_TRAP_ACCESS, regs, irqflags);
+
 	return 0;
 }
 #else					/* CONFIG_MMU */
@@ -397,6 +480,8 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 	pud_t *pud, *pud_k;
 	pmd_t *pmd, *pmd_k;
 
+	WARN_ON_ONCE(irqs_pipelined() && !hard_irqs_disabled());
+
 	if (addr < TASK_SIZE)
 		return do_page_fault(addr, fsr, regs);
 
@@ -470,7 +555,11 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 static int
 do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
+	unsigned long irqflags;
+
+	irqflags = fault_entry(ARM_TRAP_SECTION, regs);
 	do_bad_area(addr, fsr, regs);
+	fault_exit(ARM_TRAP_SECTION, regs, irqflags);
 	return 0;
 }
 #endif /* CONFIG_ARM_LPAE */
@@ -518,10 +607,12 @@ asmlinkage void
 do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
 	const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
+	unsigned long irqflags;
 
 	if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
 		return;
 
+	irqflags = fault_entry(ARM_TRAP_DABT, regs);
 	pr_alert("8<--- cut here ---\n");
 	pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
 		inf->name, fsr, addr);
@@ -529,6 +620,7 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 
 	arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
 		       fsr, 0);
+	fault_exit(ARM_TRAP_DABT, regs, irqflags);
 }
 
 void __init
@@ -548,15 +640,18 @@ asmlinkage void
 do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
 {
 	const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
+	unsigned long irqflags;
 
 	if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
 		return;
 
+	irqflags = fault_entry(ARM_TRAP_PABT, regs);
 	pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
 		inf->name, ifsr, addr);
 
 	arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
 		       ifsr, 0);
+	fault_exit(ARM_TRAP_PABT, regs, irqflags);
 }
 
 /*
diff --git a/kernel/arch/arm/vdso/datapage.S b/kernel/arch/arm/vdso/datapage.S
index 9cd73b725d..9beb76db2e 100644
--- a/kernel/arch/arm/vdso/datapage.S
+++ b/kernel/arch/arm/vdso/datapage.S
@@ -5,6 +5,8 @@
 	.align 2
 .L_vdso_data_ptr:
 	.long	_start - . - VDSO_DATA_SIZE
+.L_vdso_priv_ptr:
+	.long	_start - . - VDSO_DATA_SIZE - VDSO_PRIV_SIZE
 
 ENTRY(__get_datapage)
 	.fnstart
@@ -14,3 +16,12 @@ ENTRY(__get_datapage)
 	bx	lr
 	.fnend
 ENDPROC(__get_datapage)
+
+ENTRY(__get_privpage)
+	.fnstart
+	adr	r0, .L_vdso_priv_ptr
+	ldr	r1, [r0]
+	add	r0, r0, r1
+	bx	lr
+	.fnend
+ENDPROC(__get_privpage)
diff --git a/kernel/arch/arm/vfp/entry.S b/kernel/arch/arm/vfp/entry.S
index 27b0a1f27f..2e6680ca9a 100644
--- a/kernel/arch/arm/vfp/entry.S
+++ b/kernel/arch/arm/vfp/entry.S
@@ -23,6 +23,7 @@
 @
 ENTRY(do_vfp)
 	inc_preempt_count r10, r4
+	disable_irq_if_pipelined
  	ldr	r4, .LCvfp
 	ldr	r11, [r10, #TI_CPU]	@ CPU number
 	add	r10, r10, #TI_VFPSTATE	@ r10 = workspace
@@ -30,6 +31,7 @@ ENTRY(do_vfp)
 ENDPROC(do_vfp)
 
 ENTRY(vfp_null_entry)
+	enable_irq_if_pipelined
 	dec_preempt_count_ti r10, r4
 	ret	lr
 ENDPROC(vfp_null_entry)
diff --git a/kernel/arch/arm/vfp/vfphw.S b/kernel/arch/arm/vfp/vfphw.S
index d5837bf05a..d512f9f61b 100644
--- a/kernel/arch/arm/vfp/vfphw.S
+++ b/kernel/arch/arm/vfp/vfphw.S
@@ -170,6 +170,7 @@ vfp_hw_state_valid:
 					@ out before setting an FPEXC that
 					@ stops us reading stuff
 	VFPFMXR	FPEXC, r1		@ Restore FPEXC last
+	enable_irq_if_pipelined
 	sub	r2, r2, #4		@ Retry current instruction - if Thumb
 	str	r2, [sp, #S_PC]		@ mode it's two 16-bit instructions,
 					@ else it's one 32-bit instruction, so
@@ -199,6 +200,7 @@ skip:
 	@ Fall into hand on to next handler - appropriate coproc instr
 	@ not recognised by VFP
 
+	enable_irq_if_pipelined
 	DBGSTR	"not VFP"
 	dec_preempt_count_ti r10, r4
 	ret	lr
diff --git a/kernel/arch/arm/vfp/vfpmodule.c b/kernel/arch/arm/vfp/vfpmodule.c
index 2cb355c1b5..de099bc9be 100644
--- a/kernel/arch/arm/vfp/vfpmodule.c
+++ b/kernel/arch/arm/vfp/vfpmodule.c
@@ -14,10 +14,12 @@
 #include <linux/signal.h>
 #include <linux/sched/signal.h>
 #include <linux/smp.h>
+#include <linux/dovetail.h>
 #include <linux/init.h>
 #include <linux/uaccess.h>
 #include <linux/user.h>
 #include <linux/export.h>
+#include <linux/smp.h>
 
 #include <asm/cp15.h>
 #include <asm/cputype.h>
@@ -90,6 +92,7 @@ static void vfp_force_reload(unsigned int cpu, struct thread_info *thread)
 static void vfp_thread_flush(struct thread_info *thread)
 {
 	union vfp_state *vfp = &thread->vfpstate;
+	unsigned long flags;
 	unsigned int cpu;
 
 	/*
@@ -100,11 +103,11 @@ static void vfp_thread_flush(struct thread_info *thread)
 	 * Do this first to ensure that preemption won't overwrite our
 	 * state saving should access to the VFP be enabled at this point.
 	 */
-	cpu = get_cpu();
+	cpu = hard_get_cpu(flags);
 	if (vfp_current_hw_state[cpu] == vfp)
 		vfp_current_hw_state[cpu] = NULL;
 	fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
-	put_cpu();
+	hard_put_cpu(flags);
 
 	memset(vfp, 0, sizeof(union vfp_state));
 
@@ -119,11 +122,12 @@ static void vfp_thread_exit(struct thread_info *thread)
 {
 	/* release case: Per-thread VFP cleanup. */
 	union vfp_state *vfp = &thread->vfpstate;
-	unsigned int cpu = get_cpu();
+	unsigned long flags;
+	unsigned int cpu = hard_get_cpu(flags);
 
 	if (vfp_current_hw_state[cpu] == vfp)
 		vfp_current_hw_state[cpu] = NULL;
-	put_cpu();
+	hard_put_cpu(flags);
 }
 
 static void vfp_thread_copy(struct thread_info *thread)
@@ -159,6 +163,7 @@ static void vfp_thread_copy(struct thread_info *thread)
 static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 {
 	struct thread_info *thread = v;
+	unsigned long flags;
 	u32 fpexc;
 #ifdef CONFIG_SMP
 	unsigned int cpu;
@@ -166,6 +171,7 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 
 	switch (cmd) {
 	case THREAD_NOTIFY_SWITCH:
+		flags = hard_cond_local_irq_save();
 		fpexc = fmrx(FPEXC);
 
 #ifdef CONFIG_SMP
@@ -185,6 +191,7 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
 		 * old state.
 		 */
 		fmxr(FPEXC, fpexc & ~FPEXC_EN);
+		hard_cond_local_irq_restore(flags);
 		break;
 
 	case THREAD_NOTIFY_FLUSH:
@@ -248,7 +255,10 @@ static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_
 
 	if (exceptions == VFP_EXCEPTION_ERROR) {
 		vfp_panic("unhandled bounce", inst);
-		vfp_raise_sigfpe(FPE_FLTINV, regs);
+		if (mark_cond_trap_entry(ARM_TRAP_VFP, regs)) {
+			vfp_raise_sigfpe(FPE_FLTINV, regs);
+			mark_trap_exit(ARM_TRAP_VFP, regs);
+		}
 		return;
 	}
 
@@ -322,7 +332,7 @@ static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs)
  */
 void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 {
-	u32 fpscr, orig_fpscr, fpsid, exceptions;
+	u32 fpscr, orig_fpscr, fpsid, exceptions, next_trigger = 0;
 
 	pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
 
@@ -352,6 +362,7 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 		/*
 		 * Synchronous exception, emulate the trigger instruction
 		 */
+		hard_cond_local_irq_enable();
 		goto emulate;
 	}
 
@@ -364,7 +375,18 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 		trigger = fmrx(FPINST);
 		regs->ARM_pc -= 4;
 #endif
-	} else if (!(fpexc & FPEXC_DEX)) {
+		if (fpexc & FPEXC_FP2V) {
+			/*
+			 * The barrier() here prevents fpinst2 being read
+			 * before the condition above.
+			 */
+			barrier();
+			next_trigger = fmrx(FPINST2);
+		}
+	}
+	hard_cond_local_irq_enable();
+
+	if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) {
 		/*
 		 * Illegal combination of bits. It can be caused by an
 		 * unallocated VFP instruction but with FPSCR.IXE set and not
@@ -404,18 +426,14 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
 	if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V))
 		goto exit;
 
-	/*
-	 * The barrier() here prevents fpinst2 being read
-	 * before the condition above.
-	 */
-	barrier();
-	trigger = fmrx(FPINST2);
+	trigger = next_trigger;
 
  emulate:
 	exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
 	if (exceptions)
 		vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
  exit:
+	hard_cond_local_irq_enable();
 	preempt_enable();
 }
 
@@ -515,7 +533,8 @@ static inline void vfp_pm_init(void) { }
  */
 void vfp_sync_hwstate(struct thread_info *thread)
 {
-	unsigned int cpu = get_cpu();
+	unsigned long flags;
+	unsigned int cpu = hard_get_cpu(flags);
 
 	if (vfp_state_in_hw(cpu, thread)) {
 		u32 fpexc = fmrx(FPEXC);
@@ -528,17 +547,18 @@ void vfp_sync_hwstate(struct thread_info *thread)
 		fmxr(FPEXC, fpexc);
 	}
 
-	put_cpu();
+	hard_put_cpu(flags);
 }
 
 /* Ensure that the thread reloads the hardware VFP state on the next use. */
 void vfp_flush_hwstate(struct thread_info *thread)
 {
-	unsigned int cpu = get_cpu();
+	unsigned long flags;
+	unsigned int cpu = hard_get_cpu(flags);
 
 	vfp_force_reload(cpu, thread);
 
-	put_cpu();
+	hard_put_cpu(flags);
 }
 
 /*
diff --git a/kernel/arch/arm64/Kconfig b/kernel/arch/arm64/Kconfig
index 88ac9f0b14..d453ecedd7 100644
--- a/kernel/arch/arm64/Kconfig
+++ b/kernel/arch/arm64/Kconfig
@@ -176,6 +176,8 @@ config ARM64
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_GCC_PLUGINS
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
+	select HAVE_IRQ_PIPELINE
+	select HAVE_DOVETAIL
 	select HAVE_IRQ_TIME_ACCOUNTING
 	select HAVE_NMI
 	select HAVE_PATA_PLATFORM
@@ -1178,6 +1180,8 @@ config ARCH_ENABLE_SPLIT_PMD_PTLOCK
 config CC_HAVE_SHADOW_CALL_STACK
 	def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
 
+source "kernel/Kconfig.dovetail"
+
 config PARAVIRT
 	bool "Enable paravirtualization code"
 	help
diff --git a/kernel/arch/arm64/boot/dts/broadcom/Makefile b/kernel/arch/arm64/boot/dts/broadcom/Makefile
index cb7de8d992..7f867844b5 100644
--- a/kernel/arch/arm64/boot/dts/broadcom/Makefile
+++ b/kernel/arch/arm64/boot/dts/broadcom/Makefile
@@ -3,6 +3,7 @@ dtb-$(CONFIG_ARCH_BCM2835) += bcm2711-rpi-4-b.dtb \
 			      bcm2837-rpi-3-a-plus.dtb \
 			      bcm2837-rpi-3-b.dtb \
 			      bcm2837-rpi-3-b-plus.dtb \
+			      bcm2837-rpi-3-b-nobt.dtb \
 			      bcm2837-rpi-cm3-io3.dtb
 
 subdir-y	+= northstar2
diff --git a/kernel/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b-nobt.dts b/kernel/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b-nobt.dts
new file mode 100644
index 0000000000..43f9d0f665
--- /dev/null
+++ b/kernel/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b-nobt.dts
@@ -0,0 +1,12 @@
+/dts-v1/;
+#include "bcm2837-rpi-3-b.dts"
+
+&uart0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_gpio32>;
+};
+
+&uart1 {
+	status = "disabled";
+};
diff --git a/kernel/arch/arm64/include/asm/daifflags.h b/kernel/arch/arm64/include/asm/daifflags.h
index cfdde3a568..982c807a1d 100644
--- a/kernel/arch/arm64/include/asm/daifflags.h
+++ b/kernel/arch/arm64/include/asm/daifflags.h
@@ -12,6 +12,12 @@
 #include <asm/cpufeature.h>
 #include <asm/ptrace.h>
 
+/*
+ * irq_pipeline: DAIF masking is only used in contexts where hard
+ * interrupt masking applies, so no need to virtualize for the inband
+ * stage here (the pipeline core does assume this).
+ */
+
 #define DAIF_PROCCTX		0
 #define DAIF_PROCCTX_NOIRQ	PSR_I_BIT
 #define DAIF_ERRCTX		(PSR_I_BIT | PSR_A_BIT)
@@ -35,7 +41,7 @@ static inline void local_daif_mask(void)
 	if (system_uses_irq_prio_masking())
 		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
 
-	trace_hardirqs_off();
+	trace_hardirqs_off_pipelined();
 }
 
 static inline unsigned long local_daif_save_flags(void)
@@ -72,7 +78,7 @@ static inline void local_daif_restore(unsigned long flags)
 		!(read_sysreg(daif) & PSR_I_BIT));
 
 	if (!irq_disabled) {
-		trace_hardirqs_on();
+		trace_hardirqs_on_pipelined();
 
 		if (system_uses_irq_prio_masking()) {
 			gic_write_pmr(GIC_PRIO_IRQON);
@@ -117,7 +123,7 @@ static inline void local_daif_restore(unsigned long flags)
 	write_sysreg(flags, daif);
 
 	if (irq_disabled)
-		trace_hardirqs_off();
+		trace_hardirqs_off_pipelined();
 }
 
 /*
@@ -129,7 +135,7 @@ static inline void local_daif_inherit(struct pt_regs *regs)
 	unsigned long flags = regs->pstate & DAIF_MASK;
 
 	if (interrupts_enabled(regs))
-		trace_hardirqs_on();
+		trace_hardirqs_on_pipelined();
 
 	if (system_uses_irq_prio_masking())
 		gic_write_pmr(regs->pmr_save);
diff --git a/kernel/arch/arm64/include/asm/dovetail.h b/kernel/arch/arm64/include/asm/dovetail.h
new file mode 100644
index 0000000000..6fca696a05
--- /dev/null
+++ b/kernel/arch/arm64/include/asm/dovetail.h
@@ -0,0 +1,69 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2018 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#ifndef _ASM_ARM64_DOVETAIL_H
+#define _ASM_ARM64_DOVETAIL_H
+
+#include <asm/fpsimd.h>
+
+/* ARM64 traps */
+#define ARM64_TRAP_ACCESS	0	/* Data or instruction access exception */
+#define ARM64_TRAP_ALIGN	1	/* SP/PC alignment abort */
+#define ARM64_TRAP_SEA		2	/* Synchronous external abort */
+#define ARM64_TRAP_DEBUG	3	/* Debug trap */
+#define ARM64_TRAP_UNDI		4	/* Undefined instruction */
+#define ARM64_TRAP_UNDSE	5	/* Undefined synchronous exception */
+#define ARM64_TRAP_FPE		6	/* FPSIMD exception */
+#define ARM64_TRAP_SVE		7	/* SVE access trap */
+#define ARM64_TRAP_BTI		8	/* Branch target identification */
+
+#ifdef CONFIG_DOVETAIL
+
+static inline void arch_dovetail_exec_prepare(void)
+{ }
+
+static inline void arch_dovetail_switch_prepare(bool leave_inband)
+{ }
+
+static inline void arch_dovetail_switch_finish(bool enter_inband)
+{
+	fpsimd_restore_current_oob();
+}
+
+/*
+ * 172 is __NR_prctl from unistd32 in ARM32 mode, without #inclusion
+ * hell. At the end of the day, this number is written in stone to
+ * honor the ABI stability promise anyway.
+ */
+#define arch_dovetail_is_syscall(__nr)	\
+	(is_compat_task() ? (__nr) == 172 : (__nr) == __NR_prctl)
+
+#endif
+
+/*
+ * Pass the trap event to the companion core. Return true if running
+ * in-band afterwards.
+ */
+#define mark_cond_trap_entry(__trapnr, __regs)		\
+	({						\
+		oob_trap_notify(__trapnr, __regs);	\
+		running_inband();			\
+	})
+
+/*
+ * Pass the trap event to the companion core. We expect the current
+ * context to be running on the in-band stage upon return so that our
+ * caller can tread on common kernel code.
+ */
+#define mark_trap_entry(__trapnr, __regs)				\
+	do {								\
+		bool __ret = mark_cond_trap_entry(__trapnr, __regs);	\
+		BUG_ON(dovetail_debug() && !__ret);			\
+	} while (0)
+
+#define mark_trap_exit(__trapnr, __regs)				\
+	oob_trap_unwind(__trapnr, __regs)
+
+#endif /* _ASM_ARM64_DOVETAIL_H */
diff --git a/kernel/arch/arm64/include/asm/efi.h b/kernel/arch/arm64/include/asm/efi.h
index 16892f0d05..6f527278e0 100644
--- a/kernel/arch/arm64/include/asm/efi.h
+++ b/kernel/arch/arm64/include/asm/efi.h
@@ -115,6 +115,10 @@ static inline void efifb_setup_from_dmi(struct screen_info *si, const char *opt)
 
 static inline void efi_set_pgd(struct mm_struct *mm)
 {
+	unsigned long flags;
+
+	protect_inband_mm(flags);
+
 	__switch_mm(mm);
 
 	if (system_uses_ttbr0_pan()) {
@@ -139,6 +143,8 @@ static inline void efi_set_pgd(struct mm_struct *mm)
 			update_saved_ttbr0(current, current->active_mm);
 		}
 	}
+
+	unprotect_inband_mm(flags);
 }
 
 void efi_virtmap_load(void);
diff --git a/kernel/arch/arm64/include/asm/fpsimd.h b/kernel/arch/arm64/include/asm/fpsimd.h
index 05c9c55768..47417a0f8b 100644
--- a/kernel/arch/arm64/include/asm/fpsimd.h
+++ b/kernel/arch/arm64/include/asm/fpsimd.h
@@ -43,6 +43,7 @@ extern void fpsimd_flush_thread(void);
 extern void fpsimd_signal_preserve_current_state(void);
 extern void fpsimd_preserve_current_state(void);
 extern void fpsimd_restore_current_state(void);
+extern void fpsimd_restore_current_oob(void);
 extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
 
 extern void fpsimd_bind_task_to_cpu(void);
diff --git a/kernel/arch/arm64/include/asm/irq_pipeline.h b/kernel/arch/arm64/include/asm/irq_pipeline.h
new file mode 100644
index 0000000000..5861ab3865
--- /dev/null
+++ b/kernel/arch/arm64/include/asm/irq_pipeline.h
@@ -0,0 +1,141 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2018 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#ifndef _ASM_ARM64_IRQ_PIPELINE_H
+#define _ASM_ARM64_IRQ_PIPELINE_H
+
+#include <asm-generic/irq_pipeline.h>
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+/*
+ * In order to cope with the limited number of SGIs available to us,
+ * In-band IPI messages are multiplexed over SGI0, whereas out-of-band
+ * IPIs are directly mapped to SGI1-2.
+ */
+#define OOB_NR_IPI		2
+#define OOB_IPI_OFFSET		1 /* SGI1 */
+#define TIMER_OOB_IPI		(ipi_irq_base + OOB_IPI_OFFSET)
+#define RESCHEDULE_OOB_IPI	(TIMER_OOB_IPI + 1)
+
+extern int ipi_irq_base;
+
+static inline notrace
+unsigned long arch_irqs_virtual_to_native_flags(int stalled)
+{
+	return (!!stalled) << IRQMASK_I_POS;
+}
+
+static inline notrace
+unsigned long arch_irqs_native_to_virtual_flags(unsigned long flags)
+{
+	return (!!hard_irqs_disabled_flags(flags)) << IRQMASK_i_POS;
+}
+
+static inline notrace unsigned long arch_local_irq_save(void)
+{
+	int stalled = inband_irq_save();
+	barrier();
+	return arch_irqs_virtual_to_native_flags(stalled);
+}
+
+static inline notrace void arch_local_irq_enable(void)
+{
+	barrier();
+	inband_irq_enable();
+}
+
+static inline notrace void arch_local_irq_disable(void)
+{
+	inband_irq_disable();
+	barrier();
+}
+
+static inline notrace unsigned long arch_local_save_flags(void)
+{
+	int stalled = inband_irqs_disabled();
+	barrier();
+	return arch_irqs_virtual_to_native_flags(stalled);
+}
+
+static inline int arch_irqs_disabled_flags(unsigned long flags)
+{
+	return native_irqs_disabled_flags(flags);
+}
+
+static inline notrace void arch_local_irq_restore(unsigned long flags)
+{
+	inband_irq_restore(arch_irqs_disabled_flags(flags));
+	barrier();
+}
+
+static inline
+void arch_save_timer_regs(struct pt_regs *dst, struct pt_regs *src)
+{
+	dst->pstate = src->pstate;
+	dst->pc = src->pc;
+}
+
+static inline bool arch_steal_pipelined_tick(struct pt_regs *regs)
+{
+	return !!(regs->pstate & IRQMASK_I_BIT);
+}
+
+static inline int arch_enable_oob_stage(void)
+{
+	return 0;
+}
+
+/*
+ * We use neither the generic entry code nor
+ * kentry_enter/exit_pipelined yet. We still build a no-op version of
+ * the latter for now, until we enventually switch to using whichever
+ * of them is available first.
+ */
+#define arch_kentry_get_irqstate(__regs)	0
+
+#define arch_kentry_set_irqstate(__regs, __irqstate)	\
+	do { (void)__irqstate; } while (0)
+
+#else  /* !CONFIG_IRQ_PIPELINE */
+
+static inline unsigned long arch_local_irq_save(void)
+{
+	return native_irq_save();
+}
+
+static inline void arch_local_irq_enable(void)
+{
+	native_irq_enable();
+}
+
+static inline void arch_local_irq_disable(void)
+{
+	native_irq_disable();
+}
+
+static inline unsigned long arch_local_save_flags(void)
+{
+	return native_save_flags();
+}
+
+static inline void arch_local_irq_restore(unsigned long flags)
+{
+	native_irq_restore(flags);
+}
+
+static inline int arch_irqs_disabled_flags(unsigned long flags)
+{
+	return native_irqs_disabled_flags(flags);
+}
+
+#endif /* !CONFIG_IRQ_PIPELINE */
+
+static inline int arch_irqs_disabled(void)
+{
+	return arch_irqs_disabled_flags(arch_local_save_flags());
+}
+
+#endif /* _ASM_ARM64_IRQ_PIPELINE_H */
diff --git a/kernel/arch/arm64/include/asm/irqflags.h b/kernel/arch/arm64/include/asm/irqflags.h
index ff328e5bbb..177c7e7c5a 100644
--- a/kernel/arch/arm64/include/asm/irqflags.h
+++ b/kernel/arch/arm64/include/asm/irqflags.h
@@ -10,6 +10,10 @@
 #include <asm/ptrace.h>
 #include <asm/sysreg.h>
 
+#define IRQMASK_I_BIT	PSR_I_BIT
+#define IRQMASK_I_POS	7
+#define IRQMASK_i_POS	31
+
 /*
  * Aarch64 has flags for masking: Debug, Asynchronous (serror), Interrupts and
  * FIQ exceptions, in the 'daif' register. We mask and unmask them in 'dai'
@@ -26,7 +30,7 @@
 /*
  * CPU interrupt mask handling.
  */
-static inline void arch_local_irq_enable(void)
+static inline void native_irq_enable(void)
 {
 	if (system_has_prio_mask_debugging()) {
 		u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1);
@@ -35,7 +39,7 @@ static inline void arch_local_irq_enable(void)
 	}
 
 	asm volatile(ALTERNATIVE(
-		"msr	daifclr, #2		// arch_local_irq_enable",
+		"msr	daifclr, #2		// native_irq_enable",
 		__msr_s(SYS_ICC_PMR_EL1, "%0"),
 		ARM64_HAS_IRQ_PRIO_MASKING)
 		:
@@ -45,7 +49,7 @@ static inline void arch_local_irq_enable(void)
 	pmr_sync();
 }
 
-static inline void arch_local_irq_disable(void)
+static inline void native_irq_disable(void)
 {
 	if (system_has_prio_mask_debugging()) {
 		u32 pmr = read_sysreg_s(SYS_ICC_PMR_EL1);
@@ -54,7 +58,7 @@ static inline void arch_local_irq_disable(void)
 	}
 
 	asm volatile(ALTERNATIVE(
-		"msr	daifset, #2		// arch_local_irq_disable",
+		"msr	daifset, #2		// native_irq_disable",
 		__msr_s(SYS_ICC_PMR_EL1, "%0"),
 		ARM64_HAS_IRQ_PRIO_MASKING)
 		:
@@ -62,10 +66,17 @@ static inline void arch_local_irq_disable(void)
 		: "memory");
 }
 
+static inline void native_irq_sync(void)
+{
+	native_irq_enable();
+	isb();
+	native_irq_disable();
+}
+
 /*
  * Save the current interrupt enable state.
  */
-static inline unsigned long arch_local_save_flags(void)
+static inline unsigned long native_save_flags(void)
 {
 	unsigned long flags;
 
@@ -80,7 +91,7 @@ static inline unsigned long arch_local_save_flags(void)
 	return flags;
 }
 
-static inline int arch_irqs_disabled_flags(unsigned long flags)
+static inline int native_irqs_disabled_flags(unsigned long flags)
 {
 	int res;
 
@@ -95,23 +106,18 @@ static inline int arch_irqs_disabled_flags(unsigned long flags)
 	return res;
 }
 
-static inline int arch_irqs_disabled(void)
-{
-	return arch_irqs_disabled_flags(arch_local_save_flags());
-}
-
-static inline unsigned long arch_local_irq_save(void)
+static inline unsigned long native_irq_save(void)
 {
 	unsigned long flags;
 
-	flags = arch_local_save_flags();
+	flags = native_save_flags();
 
 	/*
 	 * There are too many states with IRQs disabled, just keep the current
 	 * state if interrupts are already disabled/masked.
 	 */
-	if (!arch_irqs_disabled_flags(flags))
-		arch_local_irq_disable();
+	if (!native_irqs_disabled_flags(flags))
+		native_irq_disable();
 
 	return flags;
 }
@@ -119,7 +125,7 @@ static inline unsigned long arch_local_irq_save(void)
 /*
  * restore saved IRQ state
  */
-static inline void arch_local_irq_restore(unsigned long flags)
+static inline void native_irq_restore(unsigned long flags)
 {
 	asm volatile(ALTERNATIVE(
 		"msr	daif, %0",
@@ -132,4 +138,12 @@ static inline void arch_local_irq_restore(unsigned long flags)
 	pmr_sync();
 }
 
+static inline bool native_irqs_disabled(void)
+{
+	unsigned long flags = native_save_flags();
+	return native_irqs_disabled_flags(flags);
+}
+
+#include <asm/irq_pipeline.h>
+
 #endif /* __ASM_IRQFLAGS_H */
diff --git a/kernel/arch/arm64/include/asm/mmu_context.h b/kernel/arch/arm64/include/asm/mmu_context.h
index cc58614884..1b0551e952 100644
--- a/kernel/arch/arm64/include/asm/mmu_context.h
+++ b/kernel/arch/arm64/include/asm/mmu_context.h
@@ -15,6 +15,7 @@
 #include <linux/sched/hotplug.h>
 #include <linux/mm_types.h>
 #include <linux/pgtable.h>
+#include <linux/irq_pipeline.h>
 
 #include <asm/cacheflush.h>
 #include <asm/cpufeature.h>
@@ -112,6 +113,9 @@ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz)
 static inline void cpu_uninstall_idmap(void)
 {
 	struct mm_struct *mm = current->active_mm;
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
 
 	cpu_set_reserved_ttbr0();
 	local_flush_tlb_all();
@@ -119,15 +123,23 @@ static inline void cpu_uninstall_idmap(void)
 
 	if (mm != &init_mm && !system_uses_ttbr0_pan())
 		cpu_switch_mm(mm->pgd, mm);
+
+	hard_cond_local_irq_restore(flags);
 }
 
 static inline void cpu_install_idmap(void)
 {
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
+
 	cpu_set_reserved_ttbr0();
 	local_flush_tlb_all();
 	cpu_set_idmap_tcr_t0sz();
 
 	cpu_switch_mm(lm_alias(idmap_pg_dir), &init_mm);
+
+	hard_cond_local_irq_restore(flags);
 }
 
 /*
@@ -230,7 +242,7 @@ static inline void __switch_mm(struct mm_struct *next)
 }
 
 static inline void
-switch_mm(struct mm_struct *prev, struct mm_struct *next,
+do_switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	  struct task_struct *tsk)
 {
 	if (prev != next)
@@ -245,8 +257,26 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	update_saved_ttbr0(tsk, next);
 }
 
+static inline void
+switch_mm(struct mm_struct *prev, struct mm_struct *next,
+	  struct task_struct *tsk)
+{
+	unsigned long flags;
+
+	protect_inband_mm(flags);
+	do_switch_mm(prev, next, tsk);
+	unprotect_inband_mm(flags);
+}
+
 #define deactivate_mm(tsk,mm)	do { } while (0)
-#define activate_mm(prev,next)	switch_mm(prev, next, current)
+#define activate_mm(prev,next)	do_switch_mm(prev, next, current)
+
+static inline void
+switch_oob_mm(struct mm_struct *prev, struct mm_struct *next,
+	      struct task_struct *tsk)
+{
+	do_switch_mm(prev, next, tsk);
+}
 
 static inline const struct cpumask *
 task_cpu_possible_mask(struct task_struct *p)
diff --git a/kernel/arch/arm64/include/asm/ptrace.h b/kernel/arch/arm64/include/asm/ptrace.h
index f834744423..ac73af4213 100644
--- a/kernel/arch/arm64/include/asm/ptrace.h
+++ b/kernel/arch/arm64/include/asm/ptrace.h
@@ -201,7 +201,13 @@ struct pt_regs {
 
 	/* Only valid for some EL1 exceptions. */
 	u64 lockdep_hardirqs;
+#ifdef CONFIG_IRQ_PIPELINE
+	u64 exit_rcu : 1,
+		oob_on_entry : 1,
+		stalled_on_entry : 1;
+#else
 	u64 exit_rcu;
+#endif
 };
 
 static inline bool in_syscall(struct pt_regs const *regs)
diff --git a/kernel/arch/arm64/include/asm/syscall.h b/kernel/arch/arm64/include/asm/syscall.h
index 03e2089545..88272178d1 100644
--- a/kernel/arch/arm64/include/asm/syscall.h
+++ b/kernel/arch/arm64/include/asm/syscall.h
@@ -73,6 +73,11 @@ static inline void syscall_get_arguments(struct task_struct *task,
 	memcpy(args, &regs->regs[1], 5 * sizeof(args[0]));
 }
 
+static inline unsigned long syscall_get_arg0(struct pt_regs *regs)
+{
+	return regs->orig_x0;
+}
+
 static inline void syscall_set_arguments(struct task_struct *task,
 					 struct pt_regs *regs,
 					 const unsigned long *args)
diff --git a/kernel/arch/arm64/include/asm/thread_info.h b/kernel/arch/arm64/include/asm/thread_info.h
index cdcf307764..1499c2c905 100644
--- a/kernel/arch/arm64/include/asm/thread_info.h
+++ b/kernel/arch/arm64/include/asm/thread_info.h
@@ -14,6 +14,7 @@
 
 struct task_struct;
 
+#include <dovetail/thread_info.h>
 #include <asm/memory.h>
 #include <asm/stack_pointer.h>
 #include <asm/types.h>
@@ -25,6 +26,7 @@ typedef unsigned long mm_segment_t;
  */
 struct thread_info {
 	unsigned long		flags;		/* low level flags */
+	unsigned long		local_flags;	/* local (synchronous) flags */
 	mm_segment_t		addr_limit;	/* address limit */
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	u64			ttbr0;		/* saved TTBR0_EL1 */
@@ -45,6 +47,7 @@ struct thread_info {
 	void			*scs_base;
 	void			*scs_sp;
 #endif
+	struct oob_thread_state	oob_state;
 };
 
 #define thread_saved_pc(tsk)	\
@@ -59,6 +62,8 @@ void arch_setup_new_exec(void);
 
 void arch_release_task_struct(struct task_struct *tsk);
 
+#define ti_local_flags(__ti)	((__ti)->local_flags)
+
 #endif
 
 #define TIF_SIGPENDING		0	/* signal pending */
@@ -69,11 +74,12 @@ void arch_release_task_struct(struct task_struct *tsk);
 #define TIF_FSCHECK		5	/* Check FS is USER_DS on return */
 #define TIF_MTE_ASYNC_FAULT	6	/* MTE Asynchronous Tag Check Fault */
 #define TIF_NOTIFY_SIGNAL	7	/* signal notifications exist */
-#define TIF_SYSCALL_TRACE	8	/* syscall trace active */
+#define TIF_SYSCALL_TRACE	13	/* syscall trace active */
 #define TIF_SYSCALL_AUDIT	9	/* syscall auditing */
 #define TIF_SYSCALL_TRACEPOINT	10	/* syscall tracepoint for ftrace */
 #define TIF_SECCOMP		11	/* syscall secure computing */
 #define TIF_SYSCALL_EMU		12	/* syscall emulation active */
+#define TIF_RETUSER		8	/* INBAND_TASK_RETUSER is pending */
 #define TIF_MEMDIE		18	/* is terminating due to OOM killer */
 #define TIF_FREEZE		19
 #define TIF_RESTORE_SIGMASK	20
@@ -83,6 +89,7 @@ void arch_release_task_struct(struct task_struct *tsk);
 #define TIF_SVE_VL_INHERIT	24	/* Inherit sve_vl_onexec across exec */
 #define TIF_SSBD		25	/* Wants SSB mitigation */
 #define TIF_TAGGED_ADDR		26	/* Allow tagged user addresses */
+#define TIF_MAYDAY		27	/* Emergency trap pending */
 
 #define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
@@ -100,11 +107,13 @@ void arch_release_task_struct(struct task_struct *tsk);
 #define _TIF_SVE		(1 << TIF_SVE)
 #define _TIF_MTE_ASYNC_FAULT	(1 << TIF_MTE_ASYNC_FAULT)
 #define _TIF_NOTIFY_SIGNAL	(1 << TIF_NOTIFY_SIGNAL)
+#define _TIF_MAYDAY		(1 << TIF_MAYDAY)
+#define _TIF_RETUSER		(1 << TIF_RETUSER)
 
 #define _TIF_WORK_MASK		(_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
 				 _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
 				 _TIF_UPROBE | _TIF_FSCHECK | _TIF_MTE_ASYNC_FAULT | \
-				 _TIF_NOTIFY_SIGNAL)
+				 _TIF_NOTIFY_SIGNAL | _TIF_RETUSER)
 
 #define _TIF_SYSCALL_WORK	(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
 				 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
@@ -126,4 +135,12 @@ void arch_release_task_struct(struct task_struct *tsk);
 	INIT_SCS							\
 }
 
+/*
+ * Local (synchronous) thread flags.
+ */
+#define _TLF_OOB		0x0001
+#define _TLF_DOVETAIL		0x0002
+#define _TLF_OFFSTAGE		0x0004
+#define _TLF_OOBTRAP		0x0008
+
 #endif /* __ASM_THREAD_INFO_H */
diff --git a/kernel/arch/arm64/include/asm/uaccess.h b/kernel/arch/arm64/include/asm/uaccess.h
index 5d0111a61c..ceb30f4ae5 100644
--- a/kernel/arch/arm64/include/asm/uaccess.h
+++ b/kernel/arch/arm64/include/asm/uaccess.h
@@ -113,7 +113,7 @@ static inline void __uaccess_ttbr0_disable(void)
 {
 	unsigned long flags, ttbr;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	ttbr = read_sysreg(ttbr1_el1);
 	ttbr &= ~TTBR_ASID_MASK;
 	/* reserved_pg_dir placed before swapper_pg_dir */
@@ -122,7 +122,7 @@ static inline void __uaccess_ttbr0_disable(void)
 	/* Set reserved ASID */
 	write_sysreg(ttbr, ttbr1_el1);
 	isb();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static inline void __uaccess_ttbr0_enable(void)
@@ -134,7 +134,7 @@ static inline void __uaccess_ttbr0_enable(void)
 	 * variable and the MSR. A context switch could trigger an ASID
 	 * roll-over and an update of 'ttbr0'.
 	 */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
 
 	/* Restore active ASID */
@@ -147,7 +147,7 @@ static inline void __uaccess_ttbr0_enable(void)
 	/* Restore user page table */
 	write_sysreg(ttbr0, ttbr0_el1);
 	isb();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static inline bool uaccess_ttbr0_disable(void)
diff --git a/kernel/arch/arm64/include/asm/vdso.h b/kernel/arch/arm64/include/asm/vdso.h
index f99dcb94b4..c63c5ac7a9 100644
--- a/kernel/arch/arm64/include/asm/vdso.h
+++ b/kernel/arch/arm64/include/asm/vdso.h
@@ -13,6 +13,11 @@
 #define VDSO_LBASE	0x0
 
 #define __VVAR_PAGES    2
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+#define __VPRIV_PAGES   1
+#else
+#define __VPRIV_PAGES   0
+#endif
 
 #ifndef __ASSEMBLY__
 
diff --git a/kernel/arch/arm64/include/asm/vdso/gettimeofday.h b/kernel/arch/arm64/include/asm/vdso/gettimeofday.h
index 4b4c0dac0e..030fe8d0fb 100644
--- a/kernel/arch/arm64/include/asm/vdso/gettimeofday.h
+++ b/kernel/arch/arm64/include/asm/vdso/gettimeofday.h
@@ -102,6 +102,71 @@ const struct vdso_data *__arch_get_timens_vdso_data(void)
 }
 #endif
 
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+
+#include <uapi/linux/fcntl.h>
+
+extern struct vdso_priv _vdso_priv; /* vdso.lds.S */
+
+static __always_inline struct vdso_priv *__arch_get_vdso_priv(void)
+{
+	return &_vdso_priv;
+}
+
+static __always_inline int clock_open_device(const char *path, int mode)
+{
+	register int  sc  asm("w8") = __NR_openat;
+	register long ret asm("x0");
+	register long x0  asm("x0") = AT_FDCWD;
+	register long x1  asm("x1") = (long)path;
+	register long x2  asm("x2") = mode;
+
+	asm volatile(
+		"svc #0\n"
+		: "=r" (ret)
+		: "r" (sc),
+		  "r" (x0), "r" (x1), "r" (x2)
+		: "cc", "memory");
+
+	return ret;
+}
+
+static __always_inline int clock_ioctl_device(int fd, unsigned int cmd, long arg)
+{
+	register int  sc  asm("w8") = __NR_ioctl;
+	register long ret asm("x0");
+	register long x0  asm("x0") = fd;
+	register long x1  asm("x1") = cmd;
+	register long x2  asm("x2") = arg;
+
+	asm volatile(
+		"svc #0\n"
+		: "=r" (ret)
+		: "r" (sc),
+		  "r" (x0), "r" (x1), "r" (x2)
+		: "cc", "memory");
+
+	return ret;
+}
+
+static __always_inline int clock_close_device(int fd)
+{
+	register int  sc  asm("w8") = __NR_close;
+	register long ret asm("x0");
+	register long x0  asm("x0") = fd;
+
+	asm volatile(
+		"svc #0\n"
+		: "=r" (ret)
+		: "r" (sc),
+		  "r" (x0)
+		: "cc", "memory");
+
+	return ret;
+}
+
+#endif	/* CONFIG_GENERIC_CLOCKSOURCE_VDSO */
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* __ASM_VDSO_GETTIMEOFDAY_H */
diff --git a/kernel/arch/arm64/kernel/Makefile b/kernel/arch/arm64/kernel/Makefile
index 64e84b7ece..6c4fd5bef9 100644
--- a/kernel/arch/arm64/kernel/Makefile
+++ b/kernel/arch/arm64/kernel/Makefile
@@ -48,6 +48,7 @@ obj-$(CONFIG_ACPI)			+= acpi.o
 obj-$(CONFIG_ACPI_NUMA)			+= acpi_numa.o
 obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL)	+= acpi_parking_protocol.o
 obj-$(CONFIG_PARAVIRT)			+= paravirt.o
+obj-$(CONFIG_IRQ_PIPELINE)		+= irq_pipeline.o
 obj-$(CONFIG_RANDOMIZE_BASE)		+= kaslr.o
 obj-$(CONFIG_HIBERNATION)		+= hibernate.o hibernate-asm.o
 obj-$(CONFIG_KEXEC_CORE)		+= machine_kexec.o relocate_kernel.o	\
diff --git a/kernel/arch/arm64/kernel/asm-offsets.c b/kernel/arch/arm64/kernel/asm-offsets.c
index 93da876a58..26b54f423c 100644
--- a/kernel/arch/arm64/kernel/asm-offsets.c
+++ b/kernel/arch/arm64/kernel/asm-offsets.c
@@ -29,6 +29,7 @@ int main(void)
   DEFINE(TSK_ACTIVE_MM,		offsetof(struct task_struct, active_mm));
   BLANK();
   DEFINE(TSK_TI_FLAGS,		offsetof(struct task_struct, thread_info.flags));
+  DEFINE(TSK_TI_LOCAL_FLAGS,	offsetof(struct task_struct, thread_info.local_flags));
   DEFINE(TSK_TI_PREEMPT,	offsetof(struct task_struct, thread_info.preempt_count));
   DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
diff --git a/kernel/arch/arm64/kernel/debug-monitors.c b/kernel/arch/arm64/kernel/debug-monitors.c
index d7f904cd00..73382fc1d3 100644
--- a/kernel/arch/arm64/kernel/debug-monitors.c
+++ b/kernel/arch/arm64/kernel/debug-monitors.c
@@ -232,7 +232,7 @@ static void send_user_sigtrap(int si_code)
 		return;
 
 	if (interrupts_enabled(regs))
-		local_irq_enable();
+		local_irq_enable_full();
 
 	arm64_force_sig_fault(SIGTRAP, si_code, instruction_pointer(regs),
 			      "User debug trap");
diff --git a/kernel/arch/arm64/kernel/entry-common.c b/kernel/arch/arm64/kernel/entry-common.c
index 64cfe4a379..120cff8ed5 100644
--- a/kernel/arch/arm64/kernel/entry-common.c
+++ b/kernel/arch/arm64/kernel/entry-common.c
@@ -8,6 +8,7 @@
 #include <linux/context_tracking.h>
 #include <linux/ptrace.h>
 #include <linux/thread_info.h>
+#include <linux/irqstage.h>
 
 #include <asm/cpufeature.h>
 #include <asm/daifflags.h>
@@ -21,7 +22,7 @@
  * This is intended to match the logic in irqentry_enter(), handling the kernel
  * mode transitions only.
  */
-static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
+static void noinstr __enter_from_kernel_mode(struct pt_regs *regs)
 {
 	regs->exit_rcu = false;
 
@@ -41,11 +42,50 @@ static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
 	mte_check_tfsr_entry();
 }
 
+static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
+{
+#ifdef CONFIG_IRQ_PIPELINE
+	/*
+	 * CAUTION: we may switch in-band as a result of handling a
+	 * trap, so if we are running out-of-band, we must make sure
+	 * not to perform the RCU exit since we did not enter it in
+	 * the first place.
+	 */
+	regs->oob_on_entry = running_oob();
+	if (regs->oob_on_entry) {
+		regs->exit_rcu = false;
+		return;
+	}
+
+	/*
+	 * We trapped from kernel space running in-band, we need to
+	 * record the virtual interrupt state into the current
+	 * register frame (regs->stalled_on_entry) in order to
+	 * reinstate it from exit_to_kernel_mode(). Next we stall the
+	 * in-band stage in order to mirror the current hardware state
+	 * (i.e. hardirqs are off).
+	 */
+	regs->stalled_on_entry = test_and_stall_inband_nocheck();
+#endif
+
+	__enter_from_kernel_mode(regs);
+
+#ifdef CONFIG_IRQ_PIPELINE
+	/*
+	 * Our caller is going to inherit the hardware interrupt state
+	 * from the trapped context once we have returned: if running
+	 * in-band, align the stall bit on the upcoming state.
+	 */
+	if (running_inband() && interrupts_enabled(regs))
+		unstall_inband_nocheck();
+#endif
+}
+
 /*
  * This is intended to match the logic in irqentry_exit(), handling the kernel
  * mode transitions only, and with preemption handled elsewhere.
  */
-static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
+static void noinstr __exit_to_kernel_mode(struct pt_regs *regs)
 {
 	lockdep_assert_irqs_disabled();
 
@@ -67,8 +107,35 @@ static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
 	}
 }
 
+/*
+ * This is intended to match the logic in irqentry_exit(), handling the kernel
+ * mode transitions only, and with preemption handled elsewhere.
+ */
+static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
+{
+	if (running_oob())
+		return;
+
+	__exit_to_kernel_mode(regs);
+
+#ifdef CONFIG_IRQ_PIPELINE
+	/*
+	 * Reinstate the virtual interrupt state which was in effect
+	 * on entry to the trap.
+	 */
+	if (!regs->oob_on_entry) {
+		if (regs->stalled_on_entry)
+			stall_inband_nocheck();
+		else
+			unstall_inband_nocheck();
+	}
+#endif
+	return;
+}
+
 void noinstr arm64_enter_nmi(struct pt_regs *regs)
 {
+	/* irq_pipeline: running this code oob is ok. */
 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
 
 	__nmi_enter();
@@ -99,18 +166,57 @@ void noinstr arm64_exit_nmi(struct pt_regs *regs)
 
 asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
 {
-	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
+	/*
+	 * IRQ pipeline: the interrupt entry is special in that we may
+	 * run the lockdep and RCU prologue/epilogue only if the IRQ
+	 * is going to be dispatched to its handler on behalf of the
+	 * current context, i.e. only if running in-band and
+	 * unstalled. If so, we also have to reconcile the hardware
+	 * and virtual interrupt states temporarily in order to run
+	 * such prologue.
+	 */
+	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) {
 		arm64_enter_nmi(regs);
-	else
-		enter_from_kernel_mode(regs);
+	} else {
+#ifdef CONFIG_IRQ_PIPELINE
+		if (running_inband()) {
+			regs->stalled_on_entry = test_inband_stall();
+			if (!regs->stalled_on_entry) {
+				stall_inband_nocheck();
+				__enter_from_kernel_mode(regs);
+				unstall_inband_nocheck();
+			}
+		}
+#else
+		__enter_from_kernel_mode(regs);
+#endif
+	}
 }
 
 asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
 {
-	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
+	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) {
 		arm64_exit_nmi(regs);
-	else
-		exit_to_kernel_mode(regs);
+	} else {
+#ifdef CONFIG_IRQ_PIPELINE
+		/*
+		 * See enter_el1_irq_or_nmi() for details. UGLY: we
+		 * also have to tell the tracer that irqs are off,
+		 * since sync_current_irq_stage() did the opposite on
+		 * exit. Hopefully, at some point arm64 will convert
+		 * to the generic entry code which exhibits a less
+		 * convoluted logic.
+		 */
+		if (running_inband() && !regs->stalled_on_entry) {
+			stall_inband_nocheck();
+			trace_hardirqs_off();
+			__exit_to_kernel_mode(regs);
+			unstall_inband_nocheck();
+		}
+#else
+		__exit_to_kernel_mode(regs);
+#endif
+	}
 }
 
 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
@@ -231,20 +337,32 @@ asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
 
 asmlinkage void noinstr enter_from_user_mode(void)
 {
-	lockdep_hardirqs_off(CALLER_ADDR0);
-	CT_WARN_ON(ct_state() != CONTEXT_USER);
-	user_exit_irqoff();
-	trace_hardirqs_off_finish();
+	if (running_inband()) {
+		lockdep_hardirqs_off(CALLER_ADDR0);
+		WARN_ON_ONCE(irq_pipeline_debug() && test_inband_stall());
+		CT_WARN_ON(ct_state() != CONTEXT_USER);
+		stall_inband_nocheck();
+		user_exit_irqoff();
+		unstall_inband_nocheck();
+		trace_hardirqs_off_finish();
+	}
 }
 
 asmlinkage void noinstr exit_to_user_mode(void)
 {
-	mte_check_tfsr_exit();
+	if (running_inband()) {
+		trace_hardirqs_on_prepare();
+		lockdep_hardirqs_on_prepare(CALLER_ADDR0);
+		user_enter_irqoff();
+		lockdep_hardirqs_on(CALLER_ADDR0);
+		unstall_inband_nocheck();
+	}
+}
 
-	trace_hardirqs_on_prepare();
-	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
-	user_enter_irqoff();
-	lockdep_hardirqs_on(CALLER_ADDR0);
+asmlinkage void noinstr enter_el0_irq(void)
+{
+	if (running_inband() && !test_inband_stall())
+		enter_from_user_mode();
 }
 
 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
diff --git a/kernel/arch/arm64/kernel/entry.S b/kernel/arch/arm64/kernel/entry.S
index 9f19e6bb48..49a7349317 100644
--- a/kernel/arch/arm64/kernel/entry.S
+++ b/kernel/arch/arm64/kernel/entry.S
@@ -39,6 +39,12 @@
 #endif
 	.endm
 
+	.macro user_exit_el0_irq
+#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS)
+	bl	enter_el0_irq
+#endif
+	.endm
+
 	.macro user_enter_irqoff
 #if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS)
 	bl	exit_to_user_mode
@@ -534,6 +540,21 @@ SYM_CODE_END(__swpan_exit_el0)
 	mov	x24, scs_sp		// preserve the original shadow stack
 #endif
 
+#ifdef CONFIG_DOVETAIL
+	/*
+	 * When the pipeline is enabled, context switches over the irq
+	 * stack are allowed (for the co-kernel), and more interrupts
+	 * can be taken over sibling stack contexts. So we need a not so
+	 * subtle way of figuring out whether the irq stack was actually
+	 * exited, which cannot depend on the current task pointer.
+	 */
+	adr_this_cpu x25, irq_nesting, x26
+	ldr	w26, [x25]
+	cmp	w26, #0
+	add	w26, w26, #1
+	str	w26, [x25]
+	b.ne	9998f
+#else
 	/*
 	 * Compare sp with the base of the task stack.
 	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
@@ -543,6 +564,7 @@ SYM_CODE_END(__swpan_exit_el0)
 	eor	x25, x25, x19
 	and	x25, x25, #~(THREAD_SIZE - 1)
 	cbnz	x25, 9998f
+#endif
 
 	ldr_this_cpu x25, irq_stack_ptr, x26
 	mov	x26, #IRQ_STACK_SIZE
@@ -563,11 +585,18 @@ SYM_CODE_END(__swpan_exit_el0)
 	 * The callee-saved regs (x19-x29) should be preserved between
 	 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
 	 * uses x20-x23 to store data for later use.
+	 * IRQ_PIPELINE: caution, we have to preserve w0.
 	 */
 	.macro	irq_stack_exit
 	mov	sp, x19
 #ifdef CONFIG_SHADOW_CALL_STACK
 	mov	scs_sp, x24
+#endif
+#ifdef CONFIG_DOVETAIL
+	adr_this_cpu x1, irq_nesting, x2
+	ldr	w2, [x1]
+	add	w2, w2, #-1
+	str	w2, [x1]
 #endif
 	.endm
 
@@ -578,7 +607,15 @@ tsk	.req	x28		// current thread_info
  * Interrupt handling.
  */
 	.macro	irq_handler, handler:req
+#ifdef CONFIG_IRQ_PIPELINE
+#	.if	\handler == handle_arch_irq
+	ldr	x1, =handle_arch_irq_pipelined
+#	.else
+#	.error	"irq_pipeline: cannot handle interrupt"
+#	.endif
+#else
 	ldr_l	x1, \handler
+#endif
 	mov	x0, sp
 	irq_stack_entry
 	blr	x1
@@ -616,6 +653,9 @@ alternative_endif
 
 	irq_handler	\handler
 
+#ifdef CONFIG_IRQ_PIPELINE
+	cbz     w0, 66f				// skip epilogue if oob or in-band stalled
+#endif
 #ifdef CONFIG_PREEMPTION
 	ldr	x24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
@@ -630,13 +670,13 @@ alternative_else_nop_endif
 	bl	arm64_preempt_schedule_irq	// irq en/disable is done inside
 1:
 #endif
-
+66:
 	mov	x0, sp
 	bl	exit_el1_irq_or_nmi
 	.endm
 
 	.macro el0_interrupt_handler, handler:req
-	user_exit_irqoff
+	user_exit_el0_irq
 	enable_da_f
 
 	tbz	x22, #55, 1f
@@ -815,6 +855,9 @@ SYM_CODE_START_LOCAL_NOALIGN(el0_irq)
 	kernel_entry 0
 el0_irq_naked:
 	el0_interrupt_handler handle_arch_irq
+#ifdef CONFIG_IRQ_PIPELINE
+	cbz	w0, fast_ret_from_el0_irq	// skip epilogue if oob
+#endif
 	b	ret_to_user
 SYM_CODE_END(el0_irq)
 
@@ -846,6 +889,11 @@ SYM_CODE_END(el0_error)
 SYM_CODE_START_LOCAL(ret_to_user)
 	disable_daif
 	gic_prio_kentry_setup tmp=x3
+#ifdef CONFIG_IRQ_PIPELINE
+	ldr	x0, [tsk, #TSK_TI_LOCAL_FLAGS]
+	tst	x0, #_TLF_OOB
+	b.ne	fast_ret_to_user
+#endif
 #ifdef CONFIG_TRACE_IRQFLAGS
 	bl	trace_hardirqs_off
 #endif
@@ -854,12 +902,22 @@ SYM_CODE_START_LOCAL(ret_to_user)
 	cbnz	x2, work_pending
 finish_ret_to_user:
 	user_enter_irqoff
+ret_to_user_naked:
 	enable_step_tsk x19, x2
 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
 	bl	stackleak_erase
 #endif
 	kernel_exit 0
 
+#ifdef CONFIG_IRQ_PIPELINE
+fast_ret_from_el0_irq:	
+ 	disable_daif
+ 	gic_prio_kentry_setup tmp=x3
+fast_ret_to_user:
+	ldr	x19, [tsk, #TSK_TI_FLAGS]
+	b	ret_to_user_naked
+#endif
+
 /*
  * Ok, we need to do extra processing, enter the slow path.
  */
diff --git a/kernel/arch/arm64/kernel/fpsimd.c b/kernel/arch/arm64/kernel/fpsimd.c
index 5335a6bd1a..175353e4d8 100644
--- a/kernel/arch/arm64/kernel/fpsimd.c
+++ b/kernel/arch/arm64/kernel/fpsimd.c
@@ -169,6 +169,42 @@ static void __get_cpu_fpsimd_context(void)
 	WARN_ON(busy);
 }
 
+static void __put_cpu_fpsimd_context(void)
+{
+	bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
+
+	WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
+}
+
+#ifdef CONFIG_DOVETAIL
+
+#define get_cpu_fpsimd_context(__flags)			\
+	do {						\
+		(__flags) = hard_preempt_disable();	\
+		__get_cpu_fpsimd_context();		\
+	} while (0)
+
+#define put_cpu_fpsimd_context(__flags)		\
+	do {					\
+		__put_cpu_fpsimd_context();	\
+		hard_preempt_enable(__flags);	\
+	} while (0)
+
+void fpsimd_restore_current_oob(void)
+{
+	/*
+	 * Restore the fpsimd context for the current task as it
+	 * resumes from dovetail_context_switch(), which always happen
+	 * on the out-of-band stage. Skip this for kernel threads
+	 * which have no such context but always bear
+	 * TIF_FOREIGN_FPSTATE.
+	 */
+	if (current->mm)
+		fpsimd_restore_current_state();
+}
+
+#else
+
 /*
  * Claim ownership of the CPU FPSIMD context for use by the calling context.
  *
@@ -178,19 +214,12 @@ static void __get_cpu_fpsimd_context(void)
  * The double-underscore version must only be called if you know the task
  * can't be preempted.
  */
-static void get_cpu_fpsimd_context(void)
-{
-	local_bh_disable();
-	__get_cpu_fpsimd_context();
-}
-
-static void __put_cpu_fpsimd_context(void)
-{
-	bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
-
-	WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
-}
-
+#define get_cpu_fpsimd_context(__flags)			\
+	do {						\
+		preempt_disable();			\
+		__get_cpu_fpsimd_context();		\
+		(void)(__flags);			\
+	} while (0)
 /*
  * Release the CPU FPSIMD context.
  *
@@ -198,12 +227,14 @@ static void __put_cpu_fpsimd_context(void)
  * previously called, with no call to put_cpu_fpsimd_context() in the
  * meantime.
  */
-static void put_cpu_fpsimd_context(void)
-{
-	__put_cpu_fpsimd_context();
-	local_bh_enable();
-}
-
+#define put_cpu_fpsimd_context(__flags)			\
+	do {						\
+		__put_cpu_fpsimd_context();		\
+		preempt_enable();			\
+		(void)(__flags);			\
+	} while (0)
+
+#endif	/* !CONFIG_DOVETAIL */
 static bool have_cpu_fpsimd_context(void)
 {
 	return !preemptible() && __this_cpu_read(fpsimd_context_busy);
@@ -283,7 +314,7 @@ static void sve_free(struct task_struct *task)
 static void task_fpsimd_load(void)
 {
 	WARN_ON(!system_supports_fpsimd());
-	WARN_ON(!have_cpu_fpsimd_context());
+	WARN_ON(!hard_irqs_disabled() && !have_cpu_fpsimd_context());
 
 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
 		sve_load_state(sve_pffr(&current->thread),
@@ -297,14 +328,14 @@ static void task_fpsimd_load(void)
  * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
  * date with respect to the CPU registers.
  */
-static void fpsimd_save(void)
+static void __fpsimd_save(void)
 {
 	struct fpsimd_last_state_struct const *last =
 		this_cpu_ptr(&fpsimd_last_state);
 	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
 
 	WARN_ON(!system_supports_fpsimd());
-	WARN_ON(!have_cpu_fpsimd_context());
+	WARN_ON(!hard_irqs_disabled() && !have_cpu_fpsimd_context());
 
 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
 		if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
@@ -326,6 +357,15 @@ static void fpsimd_save(void)
 	}
 }
 
+void fpsimd_save(void)
+{
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
+	__fpsimd_save();
+	hard_cond_local_irq_restore(flags);
+}
+
 /*
  * All vector length selection from userspace comes through here.
  * We're on a slow path, so some sanity-checks are included.
@@ -444,7 +484,7 @@ static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
  * task->thread.uw.fpsimd_state must be up to date before calling this
  * function.
  */
-static void fpsimd_to_sve(struct task_struct *task)
+static void _fpsimd_to_sve(struct task_struct *task)
 {
 	unsigned int vq;
 	void *sst = task->thread.sve_state;
@@ -457,6 +497,15 @@ static void fpsimd_to_sve(struct task_struct *task)
 	__fpsimd_to_sve(sst, fst, vq);
 }
 
+static void fpsimd_to_sve(struct task_struct *task)
+{
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
+	_fpsimd_to_sve(task);
+	hard_cond_local_irq_restore(flags);
+}
+
 /*
  * Transfer the SVE state in task->thread.sve_state to
  * task->thread.uw.fpsimd_state.
@@ -475,15 +524,20 @@ static void sve_to_fpsimd(struct task_struct *task)
 	struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
 	unsigned int i;
 	__uint128_t const *p;
+	unsigned long flags;
 
 	if (!system_supports_sve())
 		return;
 
+	flags = hard_cond_local_irq_save();
+
 	vq = sve_vq_from_vl(task->thread.sve_vl);
 	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
 		p = (__uint128_t const *)ZREG(sst, vq, i);
 		fst->vregs[i] = arm64_le128_to_cpu(*p);
 	}
+
+	hard_cond_local_irq_restore(flags);
 }
 
 #ifdef CONFIG_ARM64_SVE
@@ -584,6 +638,8 @@ void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
 int sve_set_vector_length(struct task_struct *task,
 			  unsigned long vl, unsigned long flags)
 {
+	unsigned long irqflags = 0;
+
 	if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
 				     PR_SVE_SET_VL_ONEXEC))
 		return -EINVAL;
@@ -621,9 +677,9 @@ int sve_set_vector_length(struct task_struct *task,
 	 * non-SVE thread.
 	 */
 	if (task == current) {
-		get_cpu_fpsimd_context();
+		get_cpu_fpsimd_context(irqflags);
 
-		fpsimd_save();
+		__fpsimd_save();
 	}
 
 	fpsimd_flush_task_state(task);
@@ -631,7 +687,7 @@ int sve_set_vector_length(struct task_struct *task,
 		sve_to_fpsimd(task);
 
 	if (task == current)
-		put_cpu_fpsimd_context();
+		put_cpu_fpsimd_context(irqflags);
 
 	/*
 	 * Force reallocation of task SVE state to the correct size
@@ -936,17 +992,21 @@ void fpsimd_release_task(struct task_struct *dead_task)
  */
 void do_sve_acc(unsigned int esr, struct pt_regs *regs)
 {
+	unsigned long flags;
+
+	mark_trap_entry(ARM64_TRAP_SVE, regs);
+
 	/* Even if we chose not to use SVE, the hardware could still trap: */
 	if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
 		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
-		return;
+		goto out;
 	}
 
 	sve_alloc(current);
 
-	get_cpu_fpsimd_context();
+	get_cpu_fpsimd_context(flags);
 
-	fpsimd_save();
+	__fpsimd_save();
 
 	/* Force ret_to_user to reload the registers: */
 	fpsimd_flush_task_state(current);
@@ -955,7 +1015,9 @@ void do_sve_acc(unsigned int esr, struct pt_regs *regs)
 	if (test_and_set_thread_flag(TIF_SVE))
 		WARN_ON(1); /* SVE access shouldn't have trapped */
 
-	put_cpu_fpsimd_context();
+	put_cpu_fpsimd_context(flags);
+out:
+	mark_trap_exit(ARM64_TRAP_SVE, regs);
 }
 
 /*
@@ -974,6 +1036,9 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 {
 	unsigned int si_code = FPE_FLTUNK;
 
+	if (!mark_cond_trap_entry(ARM64_TRAP_FPE, regs))
+		return;
+
 	if (esr & ESR_ELx_FP_EXC_TFV) {
 		if (esr & FPEXC_IOF)
 			si_code = FPE_FLTINV;
@@ -990,19 +1055,24 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 	send_sig_fault(SIGFPE, si_code,
 		       (void __user *)instruction_pointer(regs),
 		       current);
+
+	mark_trap_exit(ARM64_TRAP_FPE, regs);
 }
 
 void fpsimd_thread_switch(struct task_struct *next)
 {
 	bool wrong_task, wrong_cpu;
+	unsigned long flags;
 
 	if (!system_supports_fpsimd())
 		return;
 
+	flags = hard_cond_local_irq_save();
+
 	__get_cpu_fpsimd_context();
 
 	/* Save unsaved fpsimd state, if any: */
-	fpsimd_save();
+	__fpsimd_save();
 
 	/*
 	 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
@@ -1017,16 +1087,19 @@ void fpsimd_thread_switch(struct task_struct *next)
 			       wrong_task || wrong_cpu);
 
 	__put_cpu_fpsimd_context();
+
+	hard_cond_local_irq_restore(flags);
 }
 
 void fpsimd_flush_thread(void)
 {
 	int vl, supported_vl;
+	unsigned long flags;
 
 	if (!system_supports_fpsimd())
 		return;
 
-	get_cpu_fpsimd_context();
+	get_cpu_fpsimd_context(flags);
 
 	fpsimd_flush_task_state(current);
 	memset(&current->thread.uw.fpsimd_state, 0,
@@ -1067,7 +1140,7 @@ void fpsimd_flush_thread(void)
 			current->thread.sve_vl_onexec = 0;
 	}
 
-	put_cpu_fpsimd_context();
+	put_cpu_fpsimd_context(flags);
 }
 
 /*
@@ -1076,12 +1149,14 @@ void fpsimd_flush_thread(void)
  */
 void fpsimd_preserve_current_state(void)
 {
+	unsigned long flags;
+
 	if (!system_supports_fpsimd())
 		return;
 
-	get_cpu_fpsimd_context();
-	fpsimd_save();
-	put_cpu_fpsimd_context();
+	get_cpu_fpsimd_context(flags);
+	__fpsimd_save();
+	put_cpu_fpsimd_context(flags);
 }
 
 /*
@@ -1123,7 +1198,7 @@ void fpsimd_bind_task_to_cpu(void)
 	}
 }
 
-void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
+static void __fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
 			      unsigned int sve_vl)
 {
 	struct fpsimd_last_state_struct *last =
@@ -1137,6 +1212,18 @@ void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
 	last->sve_vl = sve_vl;
 }
 
+void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
+			      unsigned int sve_vl)
+{
+	unsigned long flags;
+
+	WARN_ON(!in_softirq() && !irqs_disabled());
+
+	flags = hard_cond_local_irq_save();
+	__fpsimd_bind_state_to_cpu(st, sve_state, sve_vl);
+	hard_cond_local_irq_restore(flags);
+}
+
 /*
  * Load the userland FPSIMD state of 'current' from memory, but only if the
  * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
@@ -1144,6 +1231,8 @@ void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
  */
 void fpsimd_restore_current_state(void)
 {
+	unsigned long flags;
+
 	/*
 	 * For the tasks that were created before we detected the absence of
 	 * FP/SIMD, the TIF_FOREIGN_FPSTATE could be set via fpsimd_thread_switch(),
@@ -1158,14 +1247,14 @@ void fpsimd_restore_current_state(void)
 		return;
 	}
 
-	get_cpu_fpsimd_context();
+	get_cpu_fpsimd_context(flags);
 
 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
 		task_fpsimd_load();
 		fpsimd_bind_task_to_cpu();
 	}
 
-	put_cpu_fpsimd_context();
+	put_cpu_fpsimd_context(flags);
 }
 
 /*
@@ -1175,21 +1264,23 @@ void fpsimd_restore_current_state(void)
  */
 void fpsimd_update_current_state(struct user_fpsimd_state const *state)
 {
+	unsigned long flags;
+
 	if (WARN_ON(!system_supports_fpsimd()))
 		return;
 
-	get_cpu_fpsimd_context();
+	get_cpu_fpsimd_context(flags);
 
 	current->thread.uw.fpsimd_state = *state;
 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
-		fpsimd_to_sve(current);
+		_fpsimd_to_sve(current);
 
 	task_fpsimd_load();
 	fpsimd_bind_task_to_cpu();
 
 	clear_thread_flag(TIF_FOREIGN_FPSTATE);
 
-	put_cpu_fpsimd_context();
+	put_cpu_fpsimd_context(flags);
 }
 
 /*
@@ -1239,9 +1330,9 @@ void fpsimd_save_and_flush_cpu_state(void)
 {
 	if (!system_supports_fpsimd())
 		return;
-	WARN_ON(preemptible());
+	WARN_ON(!hard_irqs_disabled() && preemptible());
 	__get_cpu_fpsimd_context();
-	fpsimd_save();
+	__fpsimd_save();
 	fpsimd_flush_cpu_state();
 	__put_cpu_fpsimd_context();
 }
@@ -1267,18 +1358,23 @@ void fpsimd_save_and_flush_cpu_state(void)
  */
 void kernel_neon_begin(void)
 {
+	unsigned long flags;
+
 	if (WARN_ON(!system_supports_fpsimd()))
 		return;
 
 	BUG_ON(!may_use_simd());
 
-	get_cpu_fpsimd_context();
+	get_cpu_fpsimd_context(flags);
 
 	/* Save unsaved fpsimd state, if any: */
-	fpsimd_save();
+	__fpsimd_save();
 
 	/* Invalidate any task state remaining in the fpsimd regs: */
 	fpsimd_flush_cpu_state();
+
+	if (dovetailing())
+		hard_cond_local_irq_restore(flags);
 }
 EXPORT_SYMBOL(kernel_neon_begin);
 
@@ -1293,10 +1389,12 @@ EXPORT_SYMBOL(kernel_neon_begin);
  */
 void kernel_neon_end(void)
 {
+	unsigned long flags = hard_local_save_flags();
+
 	if (!system_supports_fpsimd())
 		return;
 
-	put_cpu_fpsimd_context();
+	put_cpu_fpsimd_context(flags);
 }
 EXPORT_SYMBOL(kernel_neon_end);
 
@@ -1386,9 +1484,13 @@ void __efi_fpsimd_end(void)
 static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
 				  unsigned long cmd, void *v)
 {
+	unsigned long flags;
+
 	switch (cmd) {
 	case CPU_PM_ENTER:
+		flags = hard_cond_local_irq_save();
 		fpsimd_save_and_flush_cpu_state();
+		hard_cond_local_irq_restore(flags);
 		break;
 	case CPU_PM_EXIT:
 		break;
diff --git a/kernel/arch/arm64/kernel/irq.c b/kernel/arch/arm64/kernel/irq.c
index dfb1feab86..e625e14897 100644
--- a/kernel/arch/arm64/kernel/irq.c
+++ b/kernel/arch/arm64/kernel/irq.c
@@ -14,6 +14,7 @@
 #include <linux/memory.h>
 #include <linux/smp.h>
 #include <linux/hardirq.h>
+#include <linux/irq_pipeline.h>
 #include <linux/init.h>
 #include <linux/irqchip.h>
 #include <linux/kprobes.h>
@@ -28,6 +29,15 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
 
 DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+asmlinkage int notrace
+handle_arch_irq_pipelined(struct pt_regs *regs)
+{
+	return handle_irq_pipelined(regs);
+}
+
+#endif
 
 DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
 
diff --git a/kernel/arch/arm64/kernel/irq_pipeline.c b/kernel/arch/arm64/kernel/irq_pipeline.c
new file mode 100644
index 0000000000..cc1b354fb3
--- /dev/null
+++ b/kernel/arch/arm64/kernel/irq_pipeline.c
@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2018 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#include <linux/irq.h>
+#include <linux/irq_pipeline.h>
+
+/* irq_nesting tracks the interrupt nesting level for a CPU. */
+DEFINE_PER_CPU(int, irq_nesting);
+
+void arch_do_IRQ_pipelined(struct irq_desc *desc)
+{
+	struct pt_regs *regs = raw_cpu_ptr(&irq_pipeline.tick_regs);
+	unsigned int irq = irq_desc_get_irq(desc);
+
+	__handle_domain_irq(NULL, irq, false, regs);
+}
+
+void __init arch_irq_pipeline_init(void)
+{
+	/* no per-arch init. */
+}
diff --git a/kernel/arch/arm64/kernel/process.c b/kernel/arch/arm64/kernel/process.c
index c38a5ab3a6..c2b328f964 100644
--- a/kernel/arch/arm64/kernel/process.c
+++ b/kernel/arch/arm64/kernel/process.c
@@ -125,6 +125,7 @@ void noinstr arch_cpu_idle(void)
 	 * tricks
 	 */
 	cpu_do_idle();
+	hard_cond_local_irq_enable();
 	raw_local_irq_enable();
 }
 
@@ -824,8 +825,41 @@ static int __init tagged_addr_init(void)
 core_initcall(tagged_addr_init);
 #endif	/* CONFIG_ARM64_TAGGED_ADDR_ABI */
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+/*
+ * When pipelining interrupts, we have to reconcile the hardware and
+ * the virtual states. Hard irqs are off on entry while the current
+ * stage has to be unstalled: fix this up by stalling the in-band
+ * stage on entry, unstalling on exit.
+ */
+static inline void arm64_preempt_irq_enter(void)
+{
+	WARN_ON_ONCE(irq_pipeline_debug() && test_inband_stall());
+	stall_inband();
+	trace_hardirqs_off();
+}
+
+static inline void arm64_preempt_irq_exit(void)
+{
+	trace_hardirqs_on();
+	unstall_inband();
+}
+
+#else
+
+static inline void arm64_preempt_irq_enter(void)
+{ }
+
+static inline void arm64_preempt_irq_exit(void)
+{ }
+
+#endif
+
 asmlinkage void __sched arm64_preempt_schedule_irq(void)
 {
+	arm64_preempt_irq_enter();
+
 	lockdep_assert_irqs_disabled();
 
 	/*
@@ -838,6 +872,8 @@ asmlinkage void __sched arm64_preempt_schedule_irq(void)
 	 */
 	if (system_capabilities_finalized())
 		preempt_schedule_irq();
+
+	arm64_preempt_irq_exit();
 }
 
 #ifdef CONFIG_BINFMT_ELF
diff --git a/kernel/arch/arm64/kernel/signal.c b/kernel/arch/arm64/kernel/signal.c
index b6fbbd527d..3bf8eae65b 100644
--- a/kernel/arch/arm64/kernel/signal.c
+++ b/kernel/arch/arm64/kernel/signal.c
@@ -11,6 +11,7 @@
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/signal.h>
+#include <linux/irq_pipeline.h>
 #include <linux/personality.h>
 #include <linux/freezer.h>
 #include <linux/stddef.h>
@@ -914,19 +915,36 @@ static void do_signal(struct pt_regs *regs)
 	restore_saved_sigmask();
 }
 
+static inline void do_retuser(void)
+{
+	unsigned long thread_flags;
+
+	if (dovetailing()) {
+		thread_flags = current_thread_info()->flags;
+		if (thread_flags & _TIF_RETUSER)
+			inband_retuser_notify();
+	}
+}
+
 asmlinkage void do_notify_resume(struct pt_regs *regs,
 				 unsigned long thread_flags)
 {
+	WARN_ON_ONCE(irq_pipeline_debug() && running_oob());
+
+	stall_inband();
+
 	do {
 		/* Check valid user FS if needed */
 		addr_limit_user_check();
 
 		if (thread_flags & _TIF_NEED_RESCHED) {
 			/* Unmask Debug and SError for the next task */
-			local_daif_restore(DAIF_PROCCTX_NOIRQ);
+			local_daif_restore(irqs_pipelined() ? DAIF_PROCCTX :
+					DAIF_PROCCTX_NOIRQ);
 
 			schedule();
 		} else {
+			unstall_inband();
 			local_daif_restore(DAIF_PROCCTX);
 
 			if (thread_flags & _TIF_UPROBE)
@@ -948,11 +966,29 @@ asmlinkage void do_notify_resume(struct pt_regs *regs,
 
 			if (thread_flags & _TIF_FOREIGN_FPSTATE)
 				fpsimd_restore_current_state();
+
+			do_retuser();
+			/* RETUSER might have switched oob */
+			if (running_oob()) {
+				local_daif_mask();
+				return;
+			}
 		}
 
+		/*
+		 * Dovetail: we may have restored the fpsimd state for
+		 * current with no other opportunity to check for
+		 * _TIF_FOREIGN_FPSTATE until we are back running on
+		 * el0, so we must not take any interrupt until then,
+		 * otherwise we may end up resuming with some OOB
+		 * thread's fpsimd state.
+		 */
 		local_daif_mask();
+		stall_inband();
 		thread_flags = READ_ONCE(current_thread_info()->flags);
 	} while (thread_flags & _TIF_WORK_MASK);
+
+	unstall_inband();
 }
 
 unsigned long __ro_after_init signal_minsigstksz;
diff --git a/kernel/arch/arm64/kernel/smp.c b/kernel/arch/arm64/kernel/smp.c
index 53c58a210f..1c92707072 100644
--- a/kernel/arch/arm64/kernel/smp.c
+++ b/kernel/arch/arm64/kernel/smp.c
@@ -82,7 +82,7 @@ enum ipi_msg_type {
 	NR_IPI
 };
 
-static int ipi_irq_base __read_mostly;
+int ipi_irq_base __read_mostly;
 static int nr_ipi __read_mostly = NR_IPI;
 static struct irq_desc *ipi_desc[NR_IPI] __read_mostly;
 
@@ -269,6 +269,7 @@ asmlinkage notrace void secondary_start_kernel(void)
 	complete(&cpu_running);
 
 	local_daif_restore(DAIF_PROCCTX);
+	local_irq_enable_full();
 
 	/*
 	 * OK, it's off to the idle thread for us
@@ -807,6 +808,8 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
 
 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);
 
+static unsigned int get_ipi_count(unsigned int irq, unsigned int cpu);
+
 unsigned long irq_err_count;
 
 int arch_show_interrupts(struct seq_file *p, int prec)
@@ -818,7 +821,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
 		seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
 			   prec >= 4 ? " " : "");
 		for_each_online_cpu(cpu)
-			seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu));
+			seq_printf(p, "%10u ", get_ipi_count(irq, cpu));
 		seq_printf(p, "      %s\n", ipi_types[i]);
 	}
 
@@ -884,7 +887,7 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
 
 	atomic_dec(&waiting_for_crash_ipi);
 
-	local_irq_disable();
+	local_irq_disable_full();
 	sdei_mask_local_cpu();
 
 	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
@@ -896,7 +899,7 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
 }
 
 /*
- * Main handler for inter-processor interrupts
+ * Main handler for inter-processor interrupts on the in-band stage.
  */
 static void do_handle_IPI(int ipinr)
 {
@@ -956,6 +959,73 @@ static void do_handle_IPI(int ipinr)
 		trace_ipi_exit_rcuidle(ipi_types[ipinr]);
 }
 
+static void __smp_cross_call(const struct cpumask *target, unsigned int ipinr)
+{
+	trace_ipi_raise(target, ipi_types[ipinr]);
+	__ipi_send_mask(ipi_desc[ipinr], target);
+}
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+static DEFINE_PER_CPU(unsigned long, ipi_messages);
+
+static DEFINE_PER_CPU(unsigned int [NR_IPI], ipi_counts);
+
+static irqreturn_t ipi_handler(int irq, void *data)
+{
+	unsigned long *pmsg;
+	unsigned int ipinr;
+
+	/*
+	 * Decode in-band IPIs (0..NR_IPI - 1) multiplexed over
+	 * SGI0. Out-of-band IPIs (SGI1, SGI2) have their own
+	 * individual handler.
+	 */
+	pmsg = raw_cpu_ptr(&ipi_messages);
+	while (*pmsg) {
+		ipinr = ffs(*pmsg) - 1;
+		clear_bit(ipinr, pmsg);
+		__this_cpu_inc(ipi_counts[ipinr]);
+		do_handle_IPI(ipinr);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
+{
+	unsigned int cpu;
+
+	/* regular in-band IPI (multiplexed over SGI0). */
+	for_each_cpu(cpu, target)
+		set_bit(ipinr, &per_cpu(ipi_messages, cpu));
+
+	wmb();
+	__smp_cross_call(target, 0);
+}
+
+static unsigned int get_ipi_count(unsigned int irq, unsigned int cpu)
+{
+	return per_cpu(ipi_counts[irq - ipi_irq_base], cpu);
+}
+
+void irq_send_oob_ipi(unsigned int irq,
+		const struct cpumask *cpumask)
+{
+	unsigned int sgi = irq - ipi_irq_base;
+
+	if (WARN_ON(irq_pipeline_debug() &&
+		    (sgi < OOB_IPI_OFFSET ||
+		     sgi >= OOB_IPI_OFFSET + OOB_NR_IPI)))
+		return;
+
+	/* Out-of-band IPI (SGI1-2). */
+	__smp_cross_call(cpumask, sgi);
+}
+EXPORT_SYMBOL_GPL(irq_send_oob_ipi);
+
+#else
+
 static irqreturn_t ipi_handler(int irq, void *data)
 {
 	do_handle_IPI(irq - ipi_irq_base);
@@ -964,10 +1034,16 @@ static irqreturn_t ipi_handler(int irq, void *data)
 
 static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
 {
-	trace_ipi_raise(target, ipi_types[ipinr]);
-	__ipi_send_mask(ipi_desc[ipinr], target);
+	__smp_cross_call(target, ipinr);
+}
+
+static unsigned int get_ipi_count(unsigned int irq, unsigned int cpu)
+{
+	return kstat_irqs_cpu(irq, cpu);
 }
 
+#endif /* CONFIG_IRQ_PIPELINE */
+
 static void ipi_setup(int cpu)
 {
 	int i;
@@ -994,18 +1070,25 @@ static void ipi_teardown(int cpu)
 
 void __init set_smp_ipi_range(int ipi_base, int n)
 {
-	int i;
+	int i, inband_nr_ipi;
 
 	WARN_ON(n < NR_IPI);
 	nr_ipi = min(n, NR_IPI);
+	/*
+	 * irq_pipeline: the in-band stage traps SGI0 only,
+	 * over which IPI messages are mutiplexed. Other SGIs
+	 * are available for exchanging out-of-band IPIs.
+	 */
+	inband_nr_ipi = irqs_pipelined() ? 1 : nr_ipi;
 
 	for (i = 0; i < nr_ipi; i++) {
-		int err;
-
-		err = request_percpu_irq(ipi_base + i, ipi_handler,
-					 "IPI", &cpu_number);
-		WARN_ON(err);
+		if (i < inband_nr_ipi) {
+			int err;
 
+			err = request_percpu_irq(ipi_base + i, ipi_handler,
+						"IPI", &cpu_number);
+			WARN_ON(err);
+		}
 		ipi_desc[i] = irq_to_desc(ipi_base + i);
 		irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
 
diff --git a/kernel/arch/arm64/kernel/syscall.c b/kernel/arch/arm64/kernel/syscall.c
index 6f9839d1fc..24ab7372d7 100644
--- a/kernel/arch/arm64/kernel/syscall.c
+++ b/kernel/arch/arm64/kernel/syscall.c
@@ -2,6 +2,7 @@
 
 #include <linux/compiler.h>
 #include <linux/context_tracking.h>
+#include <linux/irqstage.h>
 #include <linux/errno.h>
 #include <linux/nospec.h>
 #include <linux/ptrace.h>
@@ -94,6 +95,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
 			   const syscall_fn_t syscall_table[])
 {
 	unsigned long flags = current_thread_info()->flags;
+	int ret;
 
 	regs->orig_x0 = regs->regs[0];
 	regs->syscallno = scno;
@@ -117,9 +119,18 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
 	 */
 
 	cortex_a76_erratum_1463225_svc_handler();
+	WARN_ON_ONCE(dovetail_debug() &&
+		     running_inband() && test_inband_stall());
 	local_daif_restore(DAIF_PROCCTX);
 
-	if (flags & _TIF_MTE_ASYNC_FAULT) {
+	ret = pipeline_syscall(scno, regs);
+	if (ret > 0)
+		return;
+
+	if (ret < 0)
+		goto tail_work;
+
+	if (system_supports_mte() && (flags & _TIF_MTE_ASYNC_FAULT)) {
 		/*
 		 * Process the asynchronous tag check fault before the actual
 		 * syscall. do_notify_resume() will send a signal to userspace
@@ -159,11 +170,16 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
 	 * check again. However, if we were tracing entry, then we always trace
 	 * exit regardless, as the old entry assembly did.
 	 */
+tail_work:
 	if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
 		local_daif_mask();
+		stall_inband();
 		flags = current_thread_info()->flags;
-		if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
+		if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP)) {
+			unstall_inband();
 			return;
+		}
+		unstall_inband();
 		local_daif_restore(DAIF_PROCCTX);
 	}
 
diff --git a/kernel/arch/arm64/kernel/traps.c b/kernel/arch/arm64/kernel/traps.c
index a6ba4362e3..07d36a0ec8 100644
--- a/kernel/arch/arm64/kernel/traps.c
+++ b/kernel/arch/arm64/kernel/traps.c
@@ -15,6 +15,7 @@
 #include <linux/spinlock.h>
 #include <linux/uaccess.h>
 #include <linux/hardirq.h>
+#include <linux/irqstage.h>
 #include <linux/kdebug.h>
 #include <linux/module.h>
 #include <linux/kexec.h>
@@ -113,7 +114,7 @@ static int __die(const char *str, int err, struct pt_regs *regs)
 	return ret;
 }
 
-static DEFINE_RAW_SPINLOCK(die_lock);
+static DEFINE_HARD_SPINLOCK(die_lock);
 
 /*
  * This function is protected against re-entrancy.
@@ -285,7 +286,7 @@ void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
 }
 
 static LIST_HEAD(undef_hook);
-static DEFINE_RAW_SPINLOCK(undef_lock);
+static DEFINE_HARD_SPINLOCK(undef_lock);
 
 void register_undef_hook(struct undef_hook *hook)
 {
@@ -399,6 +400,13 @@ void arm64_notify_segfault(unsigned long addr)
 
 void do_undefinstr(struct pt_regs *regs)
 {
+	/*
+	 * If the companion core did not switched us to in-band
+	 * context, we may assume that it has handled the trap.
+	 */
+	if (running_oob())
+		return;
+
 	/* check for AArch32 breakpoint instructions */
 	if (!aarch32_break_handler(regs))
 		return;
@@ -408,14 +416,18 @@ void do_undefinstr(struct pt_regs *regs)
 
 	trace_android_rvh_do_undefinstr(regs, user_mode(regs));
 	BUG_ON(!user_mode(regs));
+	mark_trap_entry(ARM64_TRAP_UNDI, regs);
 	force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
+	mark_trap_exit(ARM64_TRAP_UNDI, regs);
 }
 NOKPROBE_SYMBOL(do_undefinstr);
 
 void do_bti(struct pt_regs *regs)
 {
 	BUG_ON(!user_mode(regs));
+	mark_trap_entry(ARM64_TRAP_BTI, regs);
 	force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
+	mark_trap_exit(ARM64_TRAP_BTI, regs);
 }
 NOKPROBE_SYMBOL(do_bti);
 
@@ -485,9 +497,11 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
 		return;
 	}
 
-	if (ret)
-		arm64_notify_segfault(tagged_address);
-	else
+	if (ret) {
+		mark_trap_entry(ARM64_TRAP_ACCESS, regs);
+		arm64_notify_segfault(address);
+		mark_trap_exit(ARM64_TRAP_ACCESS, regs);
+	} else
 		arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 }
 
@@ -533,8 +547,11 @@ static void mrs_handler(unsigned int esr, struct pt_regs *regs)
 	rt = ESR_ELx_SYS64_ISS_RT(esr);
 	sysreg = esr_sys64_to_sysreg(esr);
 
-	if (do_emulate_mrs(regs, sysreg, rt) != 0)
+	if (do_emulate_mrs(regs, sysreg, rt) != 0) {
+		mark_trap_entry(ARM64_TRAP_ACCESS, regs);
 		force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
+		mark_trap_exit(ARM64_TRAP_ACCESS, regs);
+	}
 }
 
 static void wfi_handler(unsigned int esr, struct pt_regs *regs)
@@ -761,6 +778,11 @@ const char *esr_get_class_string(u32 esr)
  */
 asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
 {
+	/*
+	 * Dovetail: Same as __do_kernel_fault(), don't bother
+	 * restoring the in-band stage, this trap is fatal and we are
+	 * already walking on thin ice.
+	 */
 	arm64_enter_nmi(regs);
 
 	console_verbose();
@@ -783,11 +805,13 @@ void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
 {
 	unsigned long pc = instruction_pointer(regs);
 
+	mark_trap_entry(ARM64_TRAP_ACCESS, regs);
 	current->thread.fault_address = 0;
 	current->thread.fault_code = esr;
 
 	arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc,
 			      "Bad EL0 synchronous exception");
+	mark_trap_exit(ARM64_TRAP_ACCESS, regs);
 }
 
 #ifdef CONFIG_VMAP_STACK
diff --git a/kernel/arch/arm64/kernel/vdso.c b/kernel/arch/arm64/kernel/vdso.c
index debb8995d5..9ddd257c01 100644
--- a/kernel/arch/arm64/kernel/vdso.c
+++ b/kernel/arch/arm64/kernel/vdso.c
@@ -43,6 +43,8 @@ enum vvar_pages {
 	VVAR_NR_PAGES,
 };
 
+#define VPRIV_NR_PAGES __VPRIV_PAGES
+
 struct vdso_abi_info {
 	const char *name;
 	const char *vdso_code_start;
@@ -123,6 +125,9 @@ static int __vdso_init(enum vdso_abi abi)
 		vdso_pagelist[i] = pfn_to_page(pfn + i);
 
 	vdso_info[abi].cm->pages = vdso_pagelist;
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+	vdso_data->cs_type_seq = CLOCKSOURCE_VDSO_NONE << 16 | 1;
+#endif
 
 	return 0;
 }
@@ -243,7 +248,8 @@ static int __setup_additional_pages(enum vdso_abi abi,
 
 	vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
 	/* Be sure to map the data page */
-	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE;
+	vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE
+		+ VPRIV_NR_PAGES * PAGE_SIZE;
 
 	vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
 	if (IS_ERR_VALUE(vdso_base)) {
@@ -251,6 +257,26 @@ static int __setup_additional_pages(enum vdso_abi abi,
 		goto up_fail;
 	}
 
+	/*
+	 * Install the vDSO mappings we need:
+	 *
+	 * +----------------+
+	 * |     vpriv      |  PAGE_SIZE (private anon page if GENERIC_CLOCKSOURCE_VDSO)
+	 * |----------------|
+	 * |     vvar       |  PAGE_SIZE (shared)
+	 * |----------------|
+	 * |     text       |  text_pages * PAGE_SIZE (shared)
+	 * |        ...     |
+	 * +----------------+
+	 */
+	if (VPRIV_NR_PAGES > 0 && mmap_region(NULL, vdso_base, PAGE_SIZE,
+			VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE,
+			0, NULL) != vdso_base) {
+		ret = ERR_PTR(-EINVAL);
+		goto up_fail;
+	}
+
+	vdso_base += VPRIV_NR_PAGES * PAGE_SIZE; /* Skip private area. */
 	ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
 				       VM_READ|VM_MAYREAD|VM_PFNMAP,
 				       vdso_info[abi].dm);
diff --git a/kernel/arch/arm64/kernel/vdso/vdso.lds.S b/kernel/arch/arm64/kernel/vdso/vdso.lds.S
index b840ab1b70..93ff9fa5c0 100644
--- a/kernel/arch/arm64/kernel/vdso/vdso.lds.S
+++ b/kernel/arch/arm64/kernel/vdso/vdso.lds.S
@@ -21,6 +21,9 @@ SECTIONS
 #ifdef CONFIG_TIME_NS
 	PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
 #endif
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+	PROVIDE(_vdso_priv = _vdso_data - __VPRIV_PAGES * PAGE_SIZE);
+#endif	
 	. = VDSO_LBASE + SIZEOF_HEADERS;
 
 	.hash		: { *(.hash) }			:text
diff --git a/kernel/arch/arm64/kernel/vdso32/vdso.lds.S b/kernel/arch/arm64/kernel/vdso32/vdso.lds.S
index 3348ce5ea3..63354d19ea 100644
--- a/kernel/arch/arm64/kernel/vdso32/vdso.lds.S
+++ b/kernel/arch/arm64/kernel/vdso32/vdso.lds.S
@@ -21,6 +21,9 @@ SECTIONS
 #ifdef CONFIG_TIME_NS
 	PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE);
 #endif
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+	PROVIDE_HIDDEN(_vdso_priv = _vdso_data - __VPRIV_PAGES * PAGE_SIZE);
+#endif	
 	. = VDSO_LBASE + SIZEOF_HEADERS;
 
 	.hash		: { *(.hash) }			:text
diff --git a/kernel/arch/arm64/mm/context.c b/kernel/arch/arm64/mm/context.c
index 001737a8f3..336aca93e6 100644
--- a/kernel/arch/arm64/mm/context.c
+++ b/kernel/arch/arm64/mm/context.c
@@ -18,7 +18,7 @@
 #include <asm/tlbflush.h>
 
 static u32 asid_bits;
-static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
+static DEFINE_HARD_SPINLOCK(cpu_asid_lock);
 
 static atomic64_t asid_generation;
 static unsigned long *asid_map;
@@ -217,6 +217,9 @@ void check_and_switch_context(struct mm_struct *mm)
 	unsigned long flags;
 	unsigned int cpu;
 	u64 asid, old_active_asid;
+	bool need_flush;
+
+	WARN_ON_ONCE(dovetail_debug() && !hard_irqs_disabled());
 
 	if (system_supports_cnp())
 		cpu_set_reserved_ttbr0();
@@ -252,12 +255,14 @@ void check_and_switch_context(struct mm_struct *mm)
 	}
 
 	cpu = smp_processor_id();
-	if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending))
-		local_flush_tlb_all();
+	need_flush = cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending);
 
 	atomic64_set(this_cpu_ptr(&active_asids), asid);
 	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
 
+	if (need_flush)
+		local_flush_tlb_all();
+
 switch_mm_fastpath:
 
 	arm64_apply_bp_hardening();
diff --git a/kernel/arch/arm64/mm/fault.c b/kernel/arch/arm64/mm/fault.c
index 45e652d326..d755cbb152 100644
--- a/kernel/arch/arm64/mm/fault.c
+++ b/kernel/arch/arm64/mm/fault.c
@@ -264,11 +264,11 @@ static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
 	    (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT)
 		return false;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	asm volatile("at s1e1r, %0" :: "r" (addr));
 	isb();
 	par = read_sysreg_par();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	/*
 	 * If we now have a valid translation, treat the translation fault as
@@ -399,6 +399,12 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr,
 		msg = "paging request";
 	}
 
+	/*
+	 * Dovetail: Don't bother restoring the in-band stage in the
+	 * non-recoverable fault case, we got busted and a full stage
+	 * switch is likely to make things even worse. Try at least to
+	 * get some debug output before panicing.
+	 */
 	die_kernel_fault(msg, addr, esr, regs);
 }
 
@@ -471,8 +477,10 @@ static void do_bad_area(unsigned long far, unsigned int esr,
 	if (user_mode(regs)) {
 		const struct fault_info *inf = esr_to_fault_info(esr);
 
+		mark_trap_entry(ARM64_TRAP_ACCESS, regs);
 		set_thread_esr(addr, esr);
 		arm64_force_sig_fault(inf->sig, inf->code, far, inf->name);
+		mark_trap_exit(ARM64_TRAP_ACCESS, regs);
 	} else {
 		__do_kernel_fault(addr, esr, regs);
 	}
@@ -537,6 +545,8 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
 	if (kprobe_page_fault(regs, esr))
 		return 0;
 
+	mark_trap_entry(ARM64_TRAP_ACCESS, regs);
+
 	/*
 	 * If we're in an interrupt or have no user context, we must not take
 	 * the fault.
@@ -612,7 +622,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
 	if (fault_signal_pending(fault, regs)) {
 		if (!user_mode(regs))
 			goto no_context;
-		return 0;
+		goto out;
 	}
 
 	if (fault & VM_FAULT_RETRY) {
@@ -637,7 +647,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
 	 */
 	if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
 			      VM_FAULT_BADACCESS))))
-		return 0;
+		goto out;
 
 	/*
 	 * If we are in kernel mode at this point, we have no context to
@@ -653,7 +663,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
 		 * oom-killed).
 		 */
 		pagefault_out_of_memory();
-		return 0;
+		goto out;
 	}
 
 	inf = esr_to_fault_info(esr);
@@ -682,10 +692,12 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
 				      far, inf->name);
 	}
 
-	return 0;
+	goto out;
 
 no_context:
 	__do_kernel_fault(addr, esr, regs);
+out:
+	mark_trap_exit(ARM64_TRAP_ACCESS, regs);
 	return 0;
 }
 
@@ -731,6 +743,8 @@ static int do_sea(unsigned long far, unsigned int esr, struct pt_regs *regs)
 	const struct fault_info *inf;
 	unsigned long siaddr;
 
+	mark_trap_entry(ARM64_TRAP_SEA, regs);
+
 	inf = esr_to_fault_info(esr);
 
 	if (user_mode(regs) && apei_claim_sea(regs) == 0) {
@@ -738,7 +752,7 @@ static int do_sea(unsigned long far, unsigned int esr, struct pt_regs *regs)
 		 * APEI claimed this as a firmware-first notification.
 		 * Some processing deferred to task_work before ret_to_user().
 		 */
-		return 0;
+		goto out;
 	}
 
 	if (esr & ESR_ELx_FnV) {
@@ -753,6 +767,8 @@ static int do_sea(unsigned long far, unsigned int esr, struct pt_regs *regs)
 	}
 	trace_android_rvh_do_sea(regs, esr, siaddr, inf->name);
 	arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
+out:
+	mark_trap_exit(ARM64_TRAP_SEA, regs);
 
 	return 0;
 }
@@ -845,6 +861,8 @@ void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs)
 	if (!inf->fn(far, esr, regs))
 		return;
 
+	mark_trap_entry(ARM64_TRAP_ACCESS, regs);
+
 	if (!user_mode(regs)) {
 		pr_alert("Unhandled fault at 0x%016lx\n", addr);
 		trace_android_rvh_do_mem_abort(regs, esr, addr, inf->name);
@@ -858,6 +876,7 @@ void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs)
 	 * address to the signal handler.
 	 */
 	arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
+	mark_trap_exit(ARM64_TRAP_ACCESS, regs);
 }
 NOKPROBE_SYMBOL(do_mem_abort);
 
@@ -871,9 +890,12 @@ NOKPROBE_SYMBOL(do_el0_irq_bp_hardening);
 void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 {
 	trace_android_rvh_do_sp_pc_abort(regs, esr, addr, user_mode(regs));
+	mark_trap_entry(ARM64_TRAP_ALIGN, regs);
 
 	arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
 			 addr, esr);
+
+	mark_trap_exit(ARM64_TRAP_ALIGN, regs);
 }
 NOKPROBE_SYMBOL(do_sp_pc_abort);
 
@@ -968,6 +990,8 @@ void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
 	if (cortex_a76_erratum_1463225_debug_handler(regs))
 		return;
 
+	mark_trap_entry(ARM64_TRAP_DEBUG, regs);
+
 	debug_exception_enter(regs);
 
 	if (user_mode(regs) && !is_ttbr0_addr(pc))
@@ -978,6 +1002,8 @@ void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
 	}
 
 	debug_exception_exit(regs);
+
+	mark_trap_exit(ARM64_TRAP_DEBUG, regs);
 }
 NOKPROBE_SYMBOL(do_debug_exception);
 
diff --git a/kernel/arch/x86/Kconfig b/kernel/arch/x86/Kconfig
index 32536ff02b..3f5a5adb23 100644
--- a/kernel/arch/x86/Kconfig
+++ b/kernel/arch/x86/Kconfig
@@ -29,6 +29,8 @@ config X86_64
 	select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT
 	select ARCH_USE_CMPXCHG_LOCKREF
 	select HAVE_ARCH_SOFT_DIRTY
+	select HAVE_IRQ_PIPELINE
+	select HAVE_DOVETAIL
 	select MODULES_USE_ELF_RELA
 	select NEED_DMA_MAP_STATE
 	select SWIOTLB
@@ -208,6 +210,7 @@ config X86
 	select HAVE_MOVE_PMD
 	select HAVE_MOVE_PUD
 	select HAVE_NMI
+	select HAVE_PERCPU_PREEMPT_COUNT
 	select HAVE_OPROFILE
 	select HAVE_OPTPROBES
 	select HAVE_PCSPKR_PLATFORM
@@ -864,6 +867,7 @@ config ACRN_GUEST
 
 endif #HYPERVISOR_GUEST
 
+source "kernel/Kconfig.dovetail"
 source "arch/x86/Kconfig.cpu"
 
 config HPET_TIMER
diff --git a/kernel/arch/x86/entry/common.c b/kernel/arch/x86/entry/common.c
index 93a3122cd1..9fdc77ae7a 100644
--- a/kernel/arch/x86/entry/common.c
+++ b/kernel/arch/x86/entry/common.c
@@ -40,6 +40,15 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
 {
 	nr = syscall_enter_from_user_mode(regs, nr);
 
+	if (dovetailing()) {
+		if (nr == EXIT_SYSCALL_OOB) {
+			hard_local_irq_disable();
+			return;
+		}
+		if (nr == EXIT_SYSCALL_TAIL)
+			goto done;
+	}
+
 	instrumentation_begin();
 	if (likely(nr < NR_syscalls)) {
 		nr = array_index_nospec(nr, NR_syscalls);
@@ -53,6 +62,7 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
 #endif
 	}
 	instrumentation_end();
+done:
 	syscall_exit_to_user_mode(regs);
 }
 #endif
@@ -89,11 +99,22 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
 	 * or may not be necessary, but it matches the old asm behavior.
 	 */
 	nr = (unsigned int)syscall_enter_from_user_mode(regs, nr);
+
+	if (dovetailing()) {
+		if (nr == EXIT_SYSCALL_OOB) {
+			hard_local_irq_disable();
+			return;
+		}
+		if (nr == EXIT_SYSCALL_TAIL)
+			goto done;
+	}
+
 	instrumentation_begin();
 
 	do_syscall_32_irqs_on(regs, nr);
 
 	instrumentation_end();
+done:
 	syscall_exit_to_user_mode(regs);
 }
 
@@ -136,9 +157,20 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
 	/* The case truncates any ptrace induced syscall nr > 2^32 -1 */
 	nr = (unsigned int)syscall_enter_from_user_mode_work(regs, nr);
 
+	if (dovetailing()) {
+		if (nr == EXIT_SYSCALL_OOB) {
+			instrumentation_end();
+			hard_local_irq_disable();
+			return true;
+		}
+		if (nr == EXIT_SYSCALL_TAIL)
+			goto done;
+	}
+
 	/* Now this is just like a normal syscall. */
 	do_syscall_32_irqs_on(regs, nr);
 
+done:
 	instrumentation_end();
 	syscall_exit_to_user_mode(regs);
 	return true;
diff --git a/kernel/arch/x86/entry/entry_64.S b/kernel/arch/x86/entry/entry_64.S
index 559c82b834..1d07a1510c 100644
--- a/kernel/arch/x86/entry/entry_64.S
+++ b/kernel/arch/x86/entry/entry_64.S
@@ -417,6 +417,11 @@ SYM_CODE_END(\asmsym)
  * If hits in kernel mode then it needs to go through the paranoid
  * entry as the exception can hit any random state. No preemption
  * check on exit to keep the paranoid path simple.
+ *
+ * irq_pipeline: since those events are non-maskable in essence,
+ * we may assume NMI-type restrictions for their handlers, which
+ * means the latter may - and actually have to - run immediately
+ * regardless of the current stage.
  */
 .macro idtentry_mce_db vector asmsym cfunc
 SYM_CODE_START(\asmsym)
diff --git a/kernel/arch/x86/hyperv/hv_init.c b/kernel/arch/x86/hyperv/hv_init.c
index 01860c0d32..fd30d6c6da 100644
--- a/kernel/arch/x86/hyperv/hv_init.c
+++ b/kernel/arch/x86/hyperv/hv_init.c
@@ -156,7 +156,8 @@ static inline bool hv_reenlightenment_available(void)
 		ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT;
 }
 
-DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(HYPERV_REENLIGHTENMENT_VECTOR,
+				 sysvec_hyperv_reenlightenment)
 {
 	ack_APIC_irq();
 	inc_irq_stat(irq_hv_reenlightenment_count);
diff --git a/kernel/arch/x86/include/asm/apic.h b/kernel/arch/x86/include/asm/apic.h
index 3b4412c83e..e104d0dde4 100644
--- a/kernel/arch/x86/include/asm/apic.h
+++ b/kernel/arch/x86/include/asm/apic.h
@@ -437,7 +437,7 @@ static inline void apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) {}
 
 extern void apic_ack_irq(struct irq_data *data);
 
-static inline void ack_APIC_irq(void)
+static inline void __ack_APIC_irq(void)
 {
 	/*
 	 * ack_APIC_irq() actually gets compiled as a single instruction
@@ -446,6 +446,11 @@ static inline void ack_APIC_irq(void)
 	apic_eoi();
 }
 
+static inline void ack_APIC_irq(void)
+{
+	if (!irqs_pipelined())
+		__ack_APIC_irq();
+}
 
 static inline bool lapic_vector_set_in_irr(unsigned int vector)
 {
diff --git a/kernel/arch/x86/include/asm/dovetail.h b/kernel/arch/x86/include/asm/dovetail.h
new file mode 100644
index 0000000000..940726f16f
--- /dev/null
+++ b/kernel/arch/x86/include/asm/dovetail.h
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2019 Philippe Gerum.
+ */
+#ifndef _ASM_X86_DOVETAIL_H
+#define _ASM_X86_DOVETAIL_H
+
+#if !defined(__ASSEMBLY__) && defined(CONFIG_DOVETAIL)
+
+#include <asm/fpu/api.h>
+#include <asm/io_bitmap.h>
+
+static inline void arch_dovetail_exec_prepare(void)
+{
+	clear_thread_flag(TIF_NEED_FPU_LOAD);
+}
+
+static inline
+void arch_dovetail_switch_prepare(bool leave_inband)
+{
+	if (leave_inband)
+		fpu__suspend_inband();
+}
+
+static inline
+void arch_dovetail_switch_finish(bool enter_inband)
+{
+	unsigned int ti_work = READ_ONCE(current_thread_info()->flags);
+
+	if (unlikely(ti_work & _TIF_IO_BITMAP))
+		tss_update_io_bitmap();
+
+	if (enter_inband) {
+		fpu__resume_inband();
+	} else {
+		  if (unlikely(ti_work & _TIF_NEED_FPU_LOAD &&
+				  !(current->flags & PF_KTHREAD)))
+			  switch_fpu_return();
+	}
+}
+
+#endif
+
+#endif /* _ASM_X86_DOVETAIL_H */
diff --git a/kernel/arch/x86/include/asm/fpu/api.h b/kernel/arch/x86/include/asm/fpu/api.h
index 8b9bfaad6e..5ef1216500 100644
--- a/kernel/arch/x86/include/asm/fpu/api.h
+++ b/kernel/arch/x86/include/asm/fpu/api.h
@@ -41,16 +41,25 @@ static inline void kernel_fpu_begin(void)
  * fpu->state and set TIF_NEED_FPU_LOAD leaving CPU's FPU registers in
  * a random state.
  */
-static inline void fpregs_lock(void)
+static inline unsigned long fpregs_lock(void)
 {
-	preempt_disable();
-	local_bh_disable();
+	if (IS_ENABLED(CONFIG_IRQ_PIPELINE)) {
+		return hard_preempt_disable();
+	} else {
+		preempt_disable();
+		local_bh_disable();
+		return 0;
+	}
 }
 
-static inline void fpregs_unlock(void)
+static inline void fpregs_unlock(unsigned long flags)
 {
-	local_bh_enable();
-	preempt_enable();
+	if (IS_ENABLED(CONFIG_IRQ_PIPELINE)) {
+		hard_preempt_enable(flags);
+	} else {
+		local_bh_enable();
+		preempt_enable();
+	}
 }
 
 #ifdef CONFIG_X86_DEBUG_FPU
@@ -64,6 +73,10 @@ static inline void fpregs_assert_state_consistent(void) { }
  */
 extern void switch_fpu_return(void);
 
+/* For Dovetail context switching. */
+void fpu__suspend_inband(void);
+void fpu__resume_inband(void);
+
 /*
  * Query the presence of one or more xfeatures. Works on any legacy CPU as well.
  *
diff --git a/kernel/arch/x86/include/asm/fpu/internal.h b/kernel/arch/x86/include/asm/fpu/internal.h
index 70b9bc5403..11d31cf648 100644
--- a/kernel/arch/x86/include/asm/fpu/internal.h
+++ b/kernel/arch/x86/include/asm/fpu/internal.h
@@ -15,6 +15,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
+#include <linux/dovetail.h>
 
 #include <asm/user.h>
 #include <asm/fpu/api.h>
@@ -509,6 +510,32 @@ static inline void __fpregs_load_activate(void)
 	clear_thread_flag(TIF_NEED_FPU_LOAD);
 }
 
+#ifdef CONFIG_DOVETAIL
+
+static inline void oob_fpu_set_preempt(struct fpu *fpu)
+{
+	fpu->preempted = 1;
+}
+
+static inline void oob_fpu_clear_preempt(struct fpu *fpu)
+{
+	fpu->preempted = 0;
+}
+
+static inline bool oob_fpu_preempted(struct fpu *old_fpu)
+{
+	return old_fpu->preempted;
+}
+
+#else
+
+static inline bool oob_fpu_preempted(struct fpu *old_fpu)
+{
+	return false;
+}
+
+#endif	/* !CONFIG_DOVETAIL */
+
 /*
  * FPU state switching for scheduling.
  *
@@ -535,7 +562,9 @@ static inline void switch_fpu_prepare(struct task_struct *prev, int cpu)
 {
 	struct fpu *old_fpu = &prev->thread.fpu;
 
-	if (static_cpu_has(X86_FEATURE_FPU) && !(prev->flags & PF_KTHREAD)) {
+	if (static_cpu_has(X86_FEATURE_FPU) &&
+		!(prev->flags & PF_KTHREAD) &&
+		!oob_fpu_preempted(old_fpu)) {
 		if (!copy_fpregs_to_fpstate(old_fpu))
 			old_fpu->last_cpu = -1;
 		else
diff --git a/kernel/arch/x86/include/asm/fpu/types.h b/kernel/arch/x86/include/asm/fpu/types.h
index f5a38a5f3a..ce2bdebd63 100644
--- a/kernel/arch/x86/include/asm/fpu/types.h
+++ b/kernel/arch/x86/include/asm/fpu/types.h
@@ -329,6 +329,18 @@ struct fpu {
 	 */
 	unsigned int			last_cpu;
 
+#ifdef CONFIG_DOVETAIL
+	/*
+	 * @preempted:
+	 *
+	 * When Dovetail is enabled, this flag is set for the inband
+	 * task context saved when entering a kernel_fpu_begin/end()
+	 * section before the latter got preempted by an out-of-band
+	 * task.
+	 */
+	unsigned char			preempted : 1;
+#endif
+
 	/*
 	 * @avx512_timestamp:
 	 *
diff --git a/kernel/arch/x86/include/asm/i8259.h b/kernel/arch/x86/include/asm/i8259.h
index 89789e8c80..facf1bc68d 100644
--- a/kernel/arch/x86/include/asm/i8259.h
+++ b/kernel/arch/x86/include/asm/i8259.h
@@ -26,7 +26,7 @@ extern unsigned int cached_irq_mask;
 #define SLAVE_ICW4_DEFAULT	0x01
 #define PIC_ICW4_AEOI		2
 
-extern raw_spinlock_t i8259A_lock;
+extern hard_spinlock_t i8259A_lock;
 
 /* the PIC may need a careful delay on some platforms, hence specific calls */
 static inline unsigned char inb_pic(unsigned int port)
diff --git a/kernel/arch/x86/include/asm/idtentry.h b/kernel/arch/x86/include/asm/idtentry.h
index dc2a8b1657..c0d1f943d1 100644
--- a/kernel/arch/x86/include/asm/idtentry.h
+++ b/kernel/arch/x86/include/asm/idtentry.h
@@ -174,6 +174,56 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
 #define DECLARE_IDTENTRY_IRQ(vector, func)				\
 	DECLARE_IDTENTRY_ERRORCODE(vector, func)
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+struct irq_stage_data;
+
+struct irq_stage_data *
+handle_irq_pipelined_prepare(struct pt_regs *regs);
+
+int handle_irq_pipelined_finish(struct irq_stage_data *prevd,
+				struct pt_regs *regs);;
+
+void arch_pipeline_entry(struct pt_regs *regs, u8 vector);
+
+#define DECLARE_IDTENTRY_SYSVEC_PIPELINED(vector, func)			\
+	DECLARE_IDTENTRY_SYSVEC(vector, func);				\
+	__visible void __##func(struct pt_regs *regs)
+
+#define DEFINE_IDTENTRY_IRQ_PIPELINED(func)				\
+__visible noinstr void func(struct pt_regs *regs,			\
+			    unsigned long error_code)			\
+{									\
+	arch_pipeline_entry(regs, (u8)error_code);			\
+}									\
+static __always_inline void __##func(struct pt_regs *regs, u8 vector)
+
+/*
+ * In a pipelined model, the actual sysvec __handler() is directly
+ * instrumentable, just like it is in fact in the non-pipelined
+ * model. The indirect call via run_on_irqstack_cond() in
+ * DEFINE_IDTENTRY_SYSVEC() happens to hide the noinstr dependency
+ * from objtool in the latter case.
+ */
+#define DEFINE_IDTENTRY_SYSVEC_PIPELINED(vector, func)			\
+__visible noinstr void func(struct pt_regs *regs)			\
+{									\
+	arch_pipeline_entry(regs, vector);				\
+}									\
+									\
+__visible void __##func(struct pt_regs *regs)
+
+#define DEFINE_IDTENTRY_SYSVEC_SIMPLE_PIPELINED(vector, func)		\
+	DEFINE_IDTENTRY_SYSVEC_PIPELINED(vector, func)
+
+#else  /* !CONFIG_IRQ_PIPELINE */
+
+#define DECLARE_IDTENTRY_SYSVEC_PIPELINED(vector, func)		DECLARE_IDTENTRY_SYSVEC(vector, func)
+
+#define DEFINE_IDTENTRY_IRQ_PIPELINED(func)			DEFINE_IDTENTRY_IRQ(func)
+#define DEFINE_IDTENTRY_SYSVEC_PIPELINED(vector, func)		DEFINE_IDTENTRY_SYSVEC(func)
+#define DEFINE_IDTENTRY_SYSVEC_SIMPLE_PIPELINED(vector, func)	DEFINE_IDTENTRY_SYSVEC_SIMPLE(func)
+
 /**
  * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points
  * @func:	Function name of the entry point
@@ -205,6 +255,8 @@ __visible noinstr void func(struct pt_regs *regs,			\
 									\
 static __always_inline void __##func(struct pt_regs *regs, u8 vector)
 
+#endif	/* !CONFIG_IRQ_PIPELINE */
+
 /**
  * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points
  * @vector:	Vector number (ignored for C)
@@ -450,6 +502,9 @@ __visible noinstr void func(struct pt_regs *regs,			\
 #define DECLARE_IDTENTRY_SYSVEC(vector, func)				\
 	idtentry_sysvec vector func
 
+#define DECLARE_IDTENTRY_SYSVEC_PIPELINED(vector, func)			\
+	DECLARE_IDTENTRY_SYSVEC(vector, func)
+
 #ifdef CONFIG_X86_64
 # define DECLARE_IDTENTRY_MCE(vector, func)				\
 	idtentry_mce_db vector asm_##func func
@@ -632,21 +687,25 @@ DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER,	spurious_interrupt);
 #ifdef CONFIG_X86_LOCAL_APIC
 DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR,		sysvec_error_interrupt);
 DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR,		sysvec_spurious_apic_interrupt);
-DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR,		sysvec_apic_timer_interrupt);
-DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR,	sysvec_x86_platform_ipi);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(LOCAL_TIMER_VECTOR,		sysvec_apic_timer_interrupt);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(X86_PLATFORM_IPI_VECTOR,	sysvec_x86_platform_ipi);
 #endif
 
 #ifdef CONFIG_SMP
-DECLARE_IDTENTRY(RESCHEDULE_VECTOR,			sysvec_reschedule_ipi);
-DECLARE_IDTENTRY_SYSVEC(IRQ_MOVE_CLEANUP_VECTOR,	sysvec_irq_move_cleanup);
-DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR,			sysvec_reboot);
-DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR,	sysvec_call_function_single);
-DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR,		sysvec_call_function);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(RESCHEDULE_VECTOR,		sysvec_reschedule_ipi);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(IRQ_MOVE_CLEANUP_VECTOR,	sysvec_irq_move_cleanup);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(REBOOT_VECTOR,		sysvec_reboot);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(CALL_FUNCTION_SINGLE_VECTOR,	sysvec_call_function_single);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(CALL_FUNCTION_VECTOR,		sysvec_call_function);
+#ifdef CONFIG_IRQ_PIPELINE
+DECLARE_IDTENTRY_SYSVEC(RESCHEDULE_OOB_VECTOR,		sysvec_reschedule_oob_ipi);
+DECLARE_IDTENTRY_SYSVEC(TIMER_OOB_VECTOR,		sysvec_timer_oob_ipi);
+#endif
 #endif
 
 #ifdef CONFIG_X86_LOCAL_APIC
 # ifdef CONFIG_X86_MCE_THRESHOLD
-DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR,		sysvec_threshold);
+DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR,	sysvec_threshold);
 # endif
 
 # ifdef CONFIG_X86_MCE_AMD
@@ -658,28 +717,28 @@ DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR,		sysvec_thermal);
 # endif
 
 # ifdef CONFIG_IRQ_WORK
-DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR,		sysvec_irq_work);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(IRQ_WORK_VECTOR,	sysvec_irq_work);
 # endif
 #endif
 
 #ifdef CONFIG_HAVE_KVM
-DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR,		sysvec_kvm_posted_intr_ipi);
-DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR,	sysvec_kvm_posted_intr_wakeup_ipi);
-DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR,	sysvec_kvm_posted_intr_nested_ipi);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(POSTED_INTR_VECTOR,		sysvec_kvm_posted_intr_ipi);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(POSTED_INTR_WAKEUP_VECTOR,	sysvec_kvm_posted_intr_wakeup_ipi);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(POSTED_INTR_NESTED_VECTOR,	sysvec_kvm_posted_intr_nested_ipi);
 #endif
 
 #if IS_ENABLED(CONFIG_HYPERV)
-DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_hyperv_callback);
-DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR,	sysvec_hyperv_reenlightenment);
-DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR,	sysvec_hyperv_stimer0);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0);
 #endif
 
 #if IS_ENABLED(CONFIG_ACRN_GUEST)
-DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_acrn_hv_callback);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(HYPERVISOR_CALLBACK_VECTOR,	sysvec_acrn_hv_callback);
 #endif
 
 #ifdef CONFIG_XEN_PVHVM
-DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_xen_hvm_callback);
+DECLARE_IDTENTRY_SYSVEC_PIPELINED(HYPERVISOR_CALLBACK_VECTOR,	sysvec_xen_hvm_callback);
 #endif
 
 #ifdef CONFIG_KVM_GUEST
diff --git a/kernel/arch/x86/include/asm/irq_pipeline.h b/kernel/arch/x86/include/asm/irq_pipeline.h
new file mode 100644
index 0000000000..5fa0cce5e6
--- /dev/null
+++ b/kernel/arch/x86/include/asm/irq_pipeline.h
@@ -0,0 +1,135 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2019 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#ifndef _ASM_X86_IRQ_PIPELINE_H
+#define _ASM_X86_IRQ_PIPELINE_H
+
+#include <asm-generic/irq_pipeline.h>
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+#include <asm/ptrace.h>
+
+#define FIRST_SYSTEM_IRQ	NR_IRQS
+#define TIMER_OOB_IPI		apicm_vector_irq(TIMER_OOB_VECTOR)
+#define RESCHEDULE_OOB_IPI	apicm_vector_irq(RESCHEDULE_OOB_VECTOR)
+#define apicm_irq_vector(__irq) ((__irq) - FIRST_SYSTEM_IRQ + FIRST_SYSTEM_VECTOR)
+#define apicm_vector_irq(__vec) ((__vec) - FIRST_SYSTEM_VECTOR + FIRST_SYSTEM_IRQ)
+
+#define X86_EFLAGS_SS_BIT	31
+
+static inline notrace
+unsigned long arch_irqs_virtual_to_native_flags(int stalled)
+{
+	return (!stalled) << X86_EFLAGS_IF_BIT;
+}
+
+static inline notrace
+unsigned long arch_irqs_native_to_virtual_flags(unsigned long flags)
+{
+	return hard_irqs_disabled_flags(flags) << X86_EFLAGS_SS_BIT;
+}
+
+#ifndef CONFIG_PARAVIRT_XXL
+
+static inline notrace unsigned long arch_local_irq_save(void)
+{
+	int stalled = inband_irq_save();
+	barrier();
+	return arch_irqs_virtual_to_native_flags(stalled);
+}
+
+static inline notrace void arch_local_irq_enable(void)
+{
+	barrier();
+	inband_irq_enable();
+}
+
+static inline notrace void arch_local_irq_disable(void)
+{
+	inband_irq_disable();
+	barrier();
+}
+
+static inline notrace unsigned long arch_local_save_flags(void)
+{
+	int stalled = inband_irqs_disabled();
+	barrier();
+	return arch_irqs_virtual_to_native_flags(stalled);
+}
+
+static inline notrace void arch_local_irq_restore(unsigned long flags)
+{
+	inband_irq_restore(native_irqs_disabled_flags(flags));
+	barrier();
+}
+
+#endif /* !CONFIG_PARAVIRT_XXL */
+
+static inline
+void arch_save_timer_regs(struct pt_regs *dst, struct pt_regs *src)
+{
+	dst->flags = src->flags;
+	dst->cs = src->cs;
+	dst->ip = src->ip;
+	dst->bp = src->bp;
+	dst->ss = src->ss;
+	dst->sp = src->sp;
+}
+
+static inline bool arch_steal_pipelined_tick(struct pt_regs *regs)
+{
+	return !(regs->flags & X86_EFLAGS_IF);
+}
+
+static inline int arch_enable_oob_stage(void)
+{
+	return 0;
+}
+
+static inline void handle_arch_irq(struct pt_regs *regs)
+{ }
+
+#else /* !CONFIG_IRQ_PIPELINE */
+
+struct pt_regs;
+
+#ifndef CONFIG_PARAVIRT_XXL
+
+static inline notrace unsigned long arch_local_save_flags(void)
+{
+	return native_save_fl();
+}
+
+static inline notrace void arch_local_irq_restore(unsigned long flags)
+{
+	native_restore_fl(flags);
+}
+
+static inline notrace void arch_local_irq_disable(void)
+{
+	native_irq_disable();
+}
+
+static inline notrace void arch_local_irq_enable(void)
+{
+	native_irq_enable();
+}
+
+/*
+ * For spinlocks, etc:
+ */
+static inline notrace unsigned long arch_local_irq_save(void)
+{
+	unsigned long flags = arch_local_save_flags();
+	arch_local_irq_disable();
+	return flags;
+}
+
+#endif /* !CONFIG_PARAVIRT_XXL */
+
+#endif /* !CONFIG_IRQ_PIPELINE */
+
+#endif /* _ASM_X86_IRQ_PIPELINE_H */
diff --git a/kernel/arch/x86/include/asm/irq_stack.h b/kernel/arch/x86/include/asm/irq_stack.h
index 775816965c..58ad3c4af7 100644
--- a/kernel/arch/x86/include/asm/irq_stack.h
+++ b/kernel/arch/x86/include/asm/irq_stack.h
@@ -18,6 +18,13 @@ void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
 void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
 			   struct irq_desc *desc);
 
+/*
+ * IRQ pipeline: only in-band (soft-)irq handlers have to run on the
+ * irqstack, oob irq handlers must be lean by design therefore can run
+ * directly over the preempted context. Therefore, the guarantee that
+ * the in-band stage is currently stalled on the current CPU is enough
+ * to update irq_count atomically.
+ */
 static __always_inline void __run_on_irqstack(void (*func)(void))
 {
 	void *tos = __this_cpu_read(hardirq_stack_ptr);
diff --git a/kernel/arch/x86/include/asm/irq_vectors.h b/kernel/arch/x86/include/asm/irq_vectors.h
index 889f8b1b5b..1e51dc4850 100644
--- a/kernel/arch/x86/include/asm/irq_vectors.h
+++ b/kernel/arch/x86/include/asm/irq_vectors.h
@@ -106,10 +106,19 @@
 
 #define LOCAL_TIMER_VECTOR		0xec
 
+#ifdef CONFIG_IRQ_PIPELINE
+#define TIMER_OOB_VECTOR		0xeb
+#define RESCHEDULE_OOB_VECTOR		0xea
+#define FIRST_SYSTEM_APIC_VECTOR	RESCHEDULE_OOB_VECTOR
+#define NR_APIC_VECTORS	        	(NR_VECTORS - FIRST_SYSTEM_VECTOR)
+#else
+#define FIRST_SYSTEM_APIC_VECTOR	LOCAL_TIMER_VECTOR
+#endif
+
 #define NR_VECTORS			 256
 
 #ifdef CONFIG_X86_LOCAL_APIC
-#define FIRST_SYSTEM_VECTOR		LOCAL_TIMER_VECTOR
+#define FIRST_SYSTEM_VECTOR		FIRST_SYSTEM_APIC_VECTOR
 #else
 #define FIRST_SYSTEM_VECTOR		NR_VECTORS
 #endif
diff --git a/kernel/arch/x86/include/asm/irqflags.h b/kernel/arch/x86/include/asm/irqflags.h
index 8c86edefa1..ca2a8700b8 100644
--- a/kernel/arch/x86/include/asm/irqflags.h
+++ b/kernel/arch/x86/include/asm/irqflags.h
@@ -35,8 +35,13 @@ extern __always_inline unsigned long native_save_fl(void)
 	return flags;
 }
 
+static inline unsigned long native_save_flags(void)
+{
+	return native_save_fl();
+}
+
 extern inline void native_restore_fl(unsigned long flags);
-extern inline void native_restore_fl(unsigned long flags)
+extern __always_inline void native_restore_fl(unsigned long flags)
 {
 	asm volatile("push %0 ; popf"
 		     : /* no output */
@@ -54,6 +59,38 @@ static __always_inline void native_irq_enable(void)
 	asm volatile("sti": : :"memory");
 }
 
+static __always_inline void native_irq_sync(void)
+{
+	asm volatile("sti ; nop ; cli": : :"memory");
+}
+
+static __always_inline unsigned long native_irq_save(void)
+{
+	unsigned long flags;
+
+	flags = native_save_flags();
+
+	native_irq_disable();
+
+	return flags;
+}
+
+static __always_inline void native_irq_restore(unsigned long flags)
+{
+	return native_restore_fl(flags);
+}
+
+static __always_inline int native_irqs_disabled_flags(unsigned long flags)
+{
+	return !(flags & X86_EFLAGS_IF);
+}
+
+static __always_inline bool native_irqs_disabled(void)
+{
+	unsigned long flags = native_save_flags();
+	return native_irqs_disabled_flags(flags);
+}
+
 static inline __cpuidle void native_safe_halt(void)
 {
 	mds_idle_clear_cpu_buffers();
@@ -73,26 +110,7 @@ static inline __cpuidle void native_halt(void)
 #else
 #ifndef __ASSEMBLY__
 #include <linux/types.h>
-
-static __always_inline unsigned long arch_local_save_flags(void)
-{
-	return native_save_fl();
-}
-
-static __always_inline void arch_local_irq_restore(unsigned long flags)
-{
-	native_restore_fl(flags);
-}
-
-static __always_inline void arch_local_irq_disable(void)
-{
-	native_irq_disable();
-}
-
-static __always_inline void arch_local_irq_enable(void)
-{
-	native_irq_enable();
-}
+#include <asm/irq_pipeline.h>
 
 /*
  * Used in the idle loop; sti takes one instruction cycle
@@ -112,15 +130,6 @@ static inline __cpuidle void halt(void)
 	native_halt();
 }
 
-/*
- * For spinlocks, etc:
- */
-static __always_inline unsigned long arch_local_irq_save(void)
-{
-	unsigned long flags = arch_local_save_flags();
-	arch_local_irq_disable();
-	return flags;
-}
 #else
 
 #define ENABLE_INTERRUPTS(x)	sti
@@ -149,7 +158,7 @@ static __always_inline unsigned long arch_local_irq_save(void)
 #ifndef __ASSEMBLY__
 static __always_inline int arch_irqs_disabled_flags(unsigned long flags)
 {
-	return !(flags & X86_EFLAGS_IF);
+	return native_irqs_disabled_flags(flags);
 }
 
 static __always_inline int arch_irqs_disabled(void)
diff --git a/kernel/arch/x86/include/asm/mmu_context.h b/kernel/arch/x86/include/asm/mmu_context.h
index d98016b837..b20f7de25d 100644
--- a/kernel/arch/x86/include/asm/mmu_context.h
+++ b/kernel/arch/x86/include/asm/mmu_context.h
@@ -128,6 +128,13 @@ extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 			       struct task_struct *tsk);
 #define switch_mm_irqs_off switch_mm_irqs_off
 
+static inline void
+switch_oob_mm(struct mm_struct *prev, struct mm_struct *next,
+	      struct task_struct *tsk)
+{
+	switch_mm_irqs_off(prev, next, tsk);
+}
+
 #define activate_mm(prev, next)			\
 do {						\
 	paravirt_activate_mm((prev), (next));	\
diff --git a/kernel/arch/x86/include/asm/pgtable.h b/kernel/arch/x86/include/asm/pgtable.h
index 87de9f2d71..d4b8b8448e 100644
--- a/kernel/arch/x86/include/asm/pgtable.h
+++ b/kernel/arch/x86/include/asm/pgtable.h
@@ -137,6 +137,7 @@ static inline u32 read_pkru(void)
 static inline void write_pkru(u32 pkru)
 {
 	struct pkru_state *pk;
+	unsigned long flags;
 
 	if (!boot_cpu_has(X86_FEATURE_OSPKE))
 		return;
@@ -148,11 +149,11 @@ static inline void write_pkru(u32 pkru)
 	 * written to the CPU. The FPU restore on return to userland would
 	 * otherwise load the previous value again.
 	 */
-	fpregs_lock();
+	flags = fpregs_lock();
 	if (pk)
 		pk->pkru = pkru;
 	__write_pkru(pkru);
-	fpregs_unlock();
+	fpregs_unlock(flags);
 }
 
 static inline int pte_young(pte_t pte)
diff --git a/kernel/arch/x86/include/asm/special_insns.h b/kernel/arch/x86/include/asm/special_insns.h
index 415693f5d9..2035bbe788 100644
--- a/kernel/arch/x86/include/asm/special_insns.h
+++ b/kernel/arch/x86/include/asm/special_insns.h
@@ -138,9 +138,9 @@ static inline void native_load_gs_index(unsigned int selector)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	asm_load_gs_index(selector);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static inline unsigned long __read_cr4(void)
diff --git a/kernel/arch/x86/include/asm/syscall.h b/kernel/arch/x86/include/asm/syscall.h
index 7cbf733d11..7b2a4648f5 100644
--- a/kernel/arch/x86/include/asm/syscall.h
+++ b/kernel/arch/x86/include/asm/syscall.h
@@ -128,6 +128,11 @@ static inline void syscall_get_arguments(struct task_struct *task,
 	}
 }
 
+static inline unsigned long syscall_get_arg0(struct pt_regs *regs)
+{
+	return regs->di;
+}
+
 static inline void syscall_set_arguments(struct task_struct *task,
 					 struct pt_regs *regs,
 					 const unsigned long *args)
diff --git a/kernel/arch/x86/include/asm/thread_info.h b/kernel/arch/x86/include/asm/thread_info.h
index 012c8ee93b..ecd2701ef0 100644
--- a/kernel/arch/x86/include/asm/thread_info.h
+++ b/kernel/arch/x86/include/asm/thread_info.h
@@ -52,10 +52,12 @@
 struct task_struct;
 #include <asm/cpufeature.h>
 #include <linux/atomic.h>
+#include <dovetail/thread_info.h>
 
 struct thread_info {
 	unsigned long		flags;		/* low level flags */
 	u32			status;		/* thread synchronous flags */
+	struct oob_thread_state	oob_state;	/* co-kernel thread state */
 };
 
 #define INIT_THREAD_INFO(tsk)			\
@@ -63,6 +65,8 @@ struct thread_info {
 	.flags		= 0,			\
 }
 
+#define ti_local_flags(__ti)	((__ti)->status)
+
 #else /* !__ASSEMBLY__ */
 
 #include <asm/asm-offsets.h>
@@ -97,8 +101,10 @@ struct thread_info {
 #define TIF_MEMDIE		20	/* is terminating due to OOM killer */
 #define TIF_POLLING_NRFLAG	21	/* idle is polling for TIF_NEED_RESCHED */
 #define TIF_IO_BITMAP		22	/* uses I/O bitmap */
+#define TIF_RETUSER		23	/* INBAND_TASK_RETUSER is pending */
 #define TIF_FORCED_TF		24	/* true if TF in eflags artificially */
 #define TIF_BLOCKSTEP		25	/* set when we want DEBUGCTLMSR_BTF */
+#define TIF_MAYDAY		26	/* emergency trap pending */
 #define TIF_LAZY_MMU_UPDATES	27	/* task is updating the mmu lazily */
 #define TIF_SYSCALL_TRACEPOINT	28	/* syscall tracepoint instrumentation */
 #define TIF_ADDR32		29	/* 32-bit address space on 64 bits */
@@ -126,7 +132,9 @@ struct thread_info {
 #define _TIF_SLD		(1 << TIF_SLD)
 #define _TIF_POLLING_NRFLAG	(1 << TIF_POLLING_NRFLAG)
 #define _TIF_IO_BITMAP		(1 << TIF_IO_BITMAP)
+#define _TIF_RETUSER		(1 << TIF_RETUSER)
 #define _TIF_FORCED_TF		(1 << TIF_FORCED_TF)
+#define _TIF_MAYDAY		(1 << TIF_MAYDAY)
 #define _TIF_BLOCKSTEP		(1 << TIF_BLOCKSTEP)
 #define _TIF_LAZY_MMU_UPDATES	(1 << TIF_LAZY_MMU_UPDATES)
 #define _TIF_SYSCALL_TRACEPOINT	(1 << TIF_SYSCALL_TRACEPOINT)
@@ -226,6 +234,16 @@ static inline int arch_within_stack_frames(const void * const stack,
  * have to worry about atomic accesses.
  */
 #define TS_COMPAT		0x0002	/* 32bit syscall active (64BIT)*/
+/* bits 2 and 3 reserved for compat */
+#define TS_OOB			0x0010	/* Thread is running out-of-band */
+#define TS_DOVETAIL		0x0020  /* Dovetail notifier enabled */
+#define TS_OFFSTAGE		0x0040	/* Thread is in-flight to OOB context */
+#define TS_OOBTRAP		0x0080	/* Handling a trap from OOB context */
+
+#define _TLF_OOB		TS_OOB
+#define _TLF_DOVETAIL		TS_DOVETAIL
+#define _TLF_OFFSTAGE		TS_OFFSTAGE
+#define _TLF_OOBTRAP		TS_OOBTRAP
 
 #ifndef __ASSEMBLY__
 #ifdef CONFIG_COMPAT
diff --git a/kernel/arch/x86/include/asm/tlbflush.h b/kernel/arch/x86/include/asm/tlbflush.h
index 8c87a2e0b6..1dfab594c6 100644
--- a/kernel/arch/x86/include/asm/tlbflush.h
+++ b/kernel/arch/x86/include/asm/tlbflush.h
@@ -37,9 +37,9 @@ static inline void cr4_set_bits(unsigned long mask)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	cr4_set_bits_irqsoff(mask);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /* Clear in this cpu's CR4. */
@@ -47,9 +47,9 @@ static inline void cr4_clear_bits(unsigned long mask)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	cr4_clear_bits_irqsoff(mask);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 #ifndef MODULE
diff --git a/kernel/arch/x86/include/asm/uaccess.h b/kernel/arch/x86/include/asm/uaccess.h
index bb1430283c..25b7697ecc 100644
--- a/kernel/arch/x86/include/asm/uaccess.h
+++ b/kernel/arch/x86/include/asm/uaccess.h
@@ -44,7 +44,7 @@ static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, un
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
 static inline bool pagefault_disabled(void);
 # define WARN_ON_IN_IRQ()	\
-	WARN_ON_ONCE(!in_task() && !pagefault_disabled())
+	WARN_ON_ONCE(running_inband() && !in_task() && !pagefault_disabled())
 #else
 # define WARN_ON_IN_IRQ()
 #endif
diff --git a/kernel/arch/x86/kernel/Makefile b/kernel/arch/x86/kernel/Makefile
index c06f3a961d..74449e6ad8 100644
--- a/kernel/arch/x86/kernel/Makefile
+++ b/kernel/arch/x86/kernel/Makefile
@@ -131,6 +131,7 @@ obj-$(CONFIG_PARAVIRT_CLOCK)	+= pvclock.o
 obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o
 
 obj-$(CONFIG_JAILHOUSE_GUEST)	+= jailhouse.o
+obj-$(CONFIG_IRQ_PIPELINE)	+= irq_pipeline.o
 
 obj-$(CONFIG_EISA)		+= eisa.o
 obj-$(CONFIG_PCSPKR_PLATFORM)	+= pcspeaker.o
diff --git a/kernel/arch/x86/kernel/alternative.c b/kernel/arch/x86/kernel/alternative.c
index 92f0a97451..e0be5d4d9b 100644
--- a/kernel/arch/x86/kernel/alternative.c
+++ b/kernel/arch/x86/kernel/alternative.c
@@ -8,6 +8,7 @@
 #include <linux/list.h>
 #include <linux/stringify.h>
 #include <linux/highmem.h>
+#include <linux/irq_pipeline.h>
 #include <linux/mm.h>
 #include <linux/vmalloc.h>
 #include <linux/memory.h>
@@ -366,9 +367,9 @@ static __always_inline int optimize_nops_range(u8 *instr, u8 instrlen, int off)
 	if (nnops <= 1)
 		return nnops;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	add_nops(instr + off, nnops);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	DUMP_BYTES(instr, instrlen, "%px: [%d:%d) optimized NOPs: ", instr, off, i);
 
@@ -1065,9 +1066,9 @@ void __init_or_module text_poke_early(void *addr, const void *opcode,
 		 */
 		memcpy(addr, opcode, len);
 	} else {
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 		memcpy(addr, opcode, len);
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		sync_core();
 
 		/*
@@ -1099,6 +1100,7 @@ static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
 	temp_mm_state_t temp_state;
 
 	lockdep_assert_irqs_disabled();
+	WARN_ON_ONCE(irq_pipeline_debug() && !hard_irqs_disabled());
 
 	/*
 	 * Make sure not to be in TLB lazy mode, as otherwise we'll end up
@@ -1192,7 +1194,7 @@ static void *__text_poke(void *addr, const void *opcode, size_t len)
 	 */
 	VM_BUG_ON(!ptep);
 
-	local_irq_save(flags);
+	local_irq_save_full(flags);
 
 	pte = mk_pte(pages[0], pgprot);
 	set_pte_at(poking_mm, poking_addr, ptep, pte);
@@ -1243,7 +1245,7 @@ static void *__text_poke(void *addr, const void *opcode, size_t len)
 	 */
 	BUG_ON(memcmp(addr, opcode, len));
 
-	local_irq_restore(flags);
+	local_irq_restore_full(flags);
 	pte_unmap_unlock(ptep, ptl);
 	return addr;
 }
diff --git a/kernel/arch/x86/kernel/apic/apic.c b/kernel/arch/x86/kernel/apic/apic.c
index 1c96f2425e..8984c7928f 100644
--- a/kernel/arch/x86/kernel/apic/apic.c
+++ b/kernel/arch/x86/kernel/apic/apic.c
@@ -31,6 +31,7 @@
 #include <linux/i8253.h>
 #include <linux/dmar.h>
 #include <linux/init.h>
+#include <linux/irq.h>
 #include <linux/cpu.h>
 #include <linux/dmi.h>
 #include <linux/smp.h>
@@ -272,10 +273,10 @@ void native_apic_icr_write(u32 low, u32 id)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
 	apic_write(APIC_ICR, low);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 u64 native_apic_icr_read(void)
@@ -331,6 +332,9 @@ int lapic_get_maxlvt(void)
 static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
 {
 	unsigned int lvtt_value, tmp_value;
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
 
 	lvtt_value = LOCAL_TIMER_VECTOR;
 	if (!oneshot)
@@ -353,6 +357,8 @@ static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
 		 * According to Intel, MFENCE can do the serialization here.
 		 */
 		asm volatile("mfence" : : : "memory");
+		hard_cond_local_irq_restore(flags);
+		printk_once(KERN_DEBUG "TSC deadline timer enabled\n");
 		return;
 	}
 
@@ -366,6 +372,8 @@ static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
 
 	if (!oneshot)
 		apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
+
+	hard_cond_local_irq_restore(flags);
 }
 
 /*
@@ -471,28 +479,34 @@ static int lapic_next_event(unsigned long delta,
 static int lapic_next_deadline(unsigned long delta,
 			       struct clock_event_device *evt)
 {
+	unsigned long flags;
 	u64 tsc;
 
 	/* This MSR is special and need a special fence: */
 	weak_wrmsr_fence();
 
+	flags = hard_local_irq_save();
 	tsc = rdtsc();
 	wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
+	hard_local_irq_restore(flags);
 	return 0;
 }
 
 static int lapic_timer_shutdown(struct clock_event_device *evt)
 {
+	unsigned long flags;
 	unsigned int v;
 
 	/* Lapic used as dummy for broadcast ? */
 	if (evt->features & CLOCK_EVT_FEAT_DUMMY)
 		return 0;
 
+	flags = hard_local_irq_save();
 	v = apic_read(APIC_LVTT);
 	v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
 	apic_write(APIC_LVTT, v);
 	apic_write(APIC_TMICT, 0);
+	hard_local_irq_restore(flags);
 	return 0;
 }
 
@@ -527,6 +541,32 @@ static void lapic_timer_broadcast(const struct cpumask *mask)
 #endif
 }
 
+static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+#define LAPIC_TIMER_IRQ  apicm_vector_irq(LOCAL_TIMER_VECTOR)
+
+static irqreturn_t lapic_oob_handler(int irq, void *dev_id)
+{
+	struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
+
+	trace_local_timer_entry(LOCAL_TIMER_VECTOR);
+	clockevents_handle_event(evt);
+	trace_local_timer_exit(LOCAL_TIMER_VECTOR);
+
+	return IRQ_HANDLED;
+}
+
+static struct irqaction lapic_oob_action = {
+	.handler = lapic_oob_handler,
+	.name = "Out-of-band LAPIC timer interrupt",
+	.flags = IRQF_TIMER | IRQF_PERCPU,
+};
+
+#else
+#define LAPIC_TIMER_IRQ  -1
+#endif
 
 /*
  * The local apic timer can be used for any function which is CPU local.
@@ -534,8 +574,8 @@ static void lapic_timer_broadcast(const struct cpumask *mask)
 static struct clock_event_device lapic_clockevent = {
 	.name				= "lapic",
 	.features			= CLOCK_EVT_FEAT_PERIODIC |
-					  CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
-					  | CLOCK_EVT_FEAT_DUMMY,
+					  CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP  |
+					  CLOCK_EVT_FEAT_PIPELINE | CLOCK_EVT_FEAT_DUMMY,
 	.shift				= 32,
 	.set_state_shutdown		= lapic_timer_shutdown,
 	.set_state_periodic		= lapic_timer_set_periodic,
@@ -544,9 +584,8 @@ static struct clock_event_device lapic_clockevent = {
 	.set_next_event			= lapic_next_event,
 	.broadcast			= lapic_timer_broadcast,
 	.rating				= 100,
-	.irq				= -1,
+	.irq				= LAPIC_TIMER_IRQ,
 };
-static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
 
 static const struct x86_cpu_id deadline_match[] __initconst = {
 	X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */
@@ -1042,6 +1081,9 @@ void __init setup_boot_APIC_clock(void)
 	/* Setup the lapic or request the broadcast */
 	setup_APIC_timer();
 	amd_e400_c1e_apic_setup();
+#ifdef CONFIG_IRQ_PIPELINE
+	setup_percpu_irq(LAPIC_TIMER_IRQ, &lapic_oob_action);
+#endif
 }
 
 void setup_secondary_APIC_clock(void)
@@ -1092,7 +1134,8 @@ static void local_apic_timer_interrupt(void)
  * [ if a single-CPU system runs an SMP kernel then we call the local
  *   interrupt as well. Thus we cannot inline the local irq ... ]
  */
-DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(LOCAL_TIMER_VECTOR,
+				 sysvec_apic_timer_interrupt)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
@@ -1513,7 +1556,7 @@ static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
 		 * per set bit.
 		 */
 		for_each_set_bit(bit, isr->map, APIC_IR_BITS)
-			ack_APIC_irq();
+			__ack_APIC_irq();
 		return true;
 	}
 
@@ -2131,7 +2174,7 @@ void __init register_lapic_address(unsigned long address)
  *
  * Also called from sysvec_spurious_apic_interrupt().
  */
-DEFINE_IDTENTRY_IRQ(spurious_interrupt)
+DEFINE_IDTENTRY_IRQ_PIPELINED(spurious_interrupt)
 {
 	u32 v;
 
@@ -2157,7 +2200,7 @@ DEFINE_IDTENTRY_IRQ(spurious_interrupt)
 	if (v & (1 << (vector & 0x1f))) {
 		pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
 			vector, smp_processor_id());
-		ack_APIC_irq();
+		__ack_APIC_irq();
 	} else {
 		pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
 			vector, smp_processor_id());
@@ -2166,13 +2209,18 @@ DEFINE_IDTENTRY_IRQ(spurious_interrupt)
 	trace_spurious_apic_exit(vector);
 }
 
-DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(SPURIOUS_APIC_VECTOR,
+				 sysvec_spurious_apic_interrupt)
 {
 	__spurious_interrupt(regs, SPURIOUS_APIC_VECTOR);
 }
 
 /*
  * This interrupt should never happen with our APIC/SMP architecture
+ *
+ * irq_pipeline: same as spurious_interrupt, would run directly out of
+ * the IDT, no deferral via the interrupt log which means that only
+ * the hardware IRQ state is considered for masking.
  */
 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
 {
diff --git a/kernel/arch/x86/kernel/apic/apic_flat_64.c b/kernel/arch/x86/kernel/apic/apic_flat_64.c
index 7862b152a0..d376218107 100644
--- a/kernel/arch/x86/kernel/apic/apic_flat_64.c
+++ b/kernel/arch/x86/kernel/apic/apic_flat_64.c
@@ -52,9 +52,9 @@ static void _flat_send_IPI_mask(unsigned long mask, int vector)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	__default_send_IPI_dest_field(mask, vector, apic->dest_logical);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static void flat_send_IPI_mask(const struct cpumask *cpumask, int vector)
diff --git a/kernel/arch/x86/kernel/apic/apic_numachip.c b/kernel/arch/x86/kernel/apic/apic_numachip.c
index 35edd57f06..3522a47801 100644
--- a/kernel/arch/x86/kernel/apic/apic_numachip.c
+++ b/kernel/arch/x86/kernel/apic/apic_numachip.c
@@ -103,10 +103,10 @@ static void numachip_send_IPI_one(int cpu, int vector)
 	if (!((apicid ^ local_apicid) >> NUMACHIP_LAPIC_BITS)) {
 		unsigned long flags;
 
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 		__default_send_IPI_dest_field(apicid, vector,
 			APIC_DEST_PHYSICAL);
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		preempt_enable();
 		return;
 	}
diff --git a/kernel/arch/x86/kernel/apic/io_apic.c b/kernel/arch/x86/kernel/apic/io_apic.c
index 25b1d5c6af..47f9d4ad36 100644
--- a/kernel/arch/x86/kernel/apic/io_apic.c
+++ b/kernel/arch/x86/kernel/apic/io_apic.c
@@ -78,7 +78,7 @@
 #define for_each_irq_pin(entry, head) \
 	list_for_each_entry(entry, &head, list)
 
-static DEFINE_RAW_SPINLOCK(ioapic_lock);
+static DEFINE_HARD_SPINLOCK(ioapic_lock);
 static DEFINE_MUTEX(ioapic_mutex);
 static unsigned int ioapic_dynirq_base;
 static int ioapic_initialized;
@@ -1634,7 +1634,7 @@ static int __init timer_irq_works(void)
 		return 1;
 
 	local_save_flags(flags);
-	local_irq_enable();
+	local_irq_enable_full();
 
 	if (boot_cpu_has(X86_FEATURE_TSC))
 		delay_with_tsc();
@@ -1642,6 +1642,8 @@ static int __init timer_irq_works(void)
 		delay_without_tsc();
 
 	local_irq_restore(flags);
+	if (raw_irqs_disabled_flags(flags))
+		hard_local_irq_disable();
 
 	/*
 	 * Expect a few ticks at least, to be sure some possible
@@ -1722,14 +1724,56 @@ static bool io_apic_level_ack_pending(struct mp_chip_data *data)
 	return false;
 }
 
+static inline void do_prepare_move(struct irq_data *data)
+{
+	if (!irqd_irq_masked(data))
+		mask_ioapic_irq(data);
+}
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+static inline void ioapic_finish_move(struct irq_data *data, bool moveit);
+
+static void ioapic_deferred_irq_move(struct irq_work *work)
+{
+	struct irq_data *data;
+	struct irq_desc *desc;
+	unsigned long flags;
+
+	data = container_of(work, struct irq_data, move_work);
+	desc = irq_data_to_desc(data);
+	raw_spin_lock_irqsave(&desc->lock, flags);
+	do_prepare_move(data);
+	ioapic_finish_move(data, true);
+	raw_spin_unlock_irqrestore(&desc->lock, flags);
+}
+
+static inline bool __ioapic_prepare_move(struct irq_data *data)
+{
+	init_irq_work(&data->move_work, ioapic_deferred_irq_move);
+	irq_work_queue(&data->move_work);
+
+	return false;	/* Postpone ioapic_finish_move(). */
+}
+
+#else  /* !CONFIG_IRQ_PIPELINE */
+
+static inline bool __ioapic_prepare_move(struct irq_data *data)
+{
+	do_prepare_move(data);
+
+	return true;
+}
+
+#endif
+
 static inline bool ioapic_prepare_move(struct irq_data *data)
 {
 	/* If we are moving the IRQ we need to mask it */
-	if (unlikely(irqd_is_setaffinity_pending(data))) {
-		if (!irqd_irq_masked(data))
-			mask_ioapic_irq(data);
-		return true;
-	}
+	if (irqd_is_setaffinity_pending(data) &&
+		!irqd_is_setaffinity_blocked(data))
+		return __ioapic_prepare_move(data);
+
 	return false;
 }
 
@@ -1828,7 +1872,7 @@ static void ioapic_ack_level(struct irq_data *irq_data)
 	 * We must acknowledge the irq before we move it or the acknowledge will
 	 * not propagate properly.
 	 */
-	ack_APIC_irq();
+	__ack_APIC_irq();
 
 	/*
 	 * Tail end of clearing remote IRR bit (either by delivering the EOI
@@ -1949,7 +1993,8 @@ static struct irq_chip ioapic_chip __read_mostly = {
 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
 	.irq_get_irqchip_state	= ioapic_irq_get_chip_state,
 	.flags			= IRQCHIP_SKIP_SET_WAKE |
-				  IRQCHIP_AFFINITY_PRE_STARTUP,
+				  IRQCHIP_AFFINITY_PRE_STARTUP |
+				  IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct irq_chip ioapic_ir_chip __read_mostly = {
@@ -1963,7 +2008,8 @@ static struct irq_chip ioapic_ir_chip __read_mostly = {
 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
 	.irq_get_irqchip_state	= ioapic_irq_get_chip_state,
 	.flags			= IRQCHIP_SKIP_SET_WAKE |
-				  IRQCHIP_AFFINITY_PRE_STARTUP,
+				  IRQCHIP_AFFINITY_PRE_STARTUP |
+				  IRQCHIP_PIPELINE_SAFE,
 };
 
 static inline void init_IO_APIC_traps(void)
@@ -2010,7 +2056,7 @@ static void unmask_lapic_irq(struct irq_data *data)
 
 static void ack_lapic_irq(struct irq_data *data)
 {
-	ack_APIC_irq();
+	__ack_APIC_irq();
 }
 
 static struct irq_chip lapic_chip __read_mostly = {
@@ -2018,6 +2064,7 @@ static struct irq_chip lapic_chip __read_mostly = {
 	.irq_mask	= mask_lapic_irq,
 	.irq_unmask	= unmask_lapic_irq,
 	.irq_ack	= ack_lapic_irq,
+	.flags		= IRQCHIP_PIPELINE_SAFE,
 };
 
 static void lapic_register_intr(int irq)
@@ -2135,7 +2182,7 @@ static inline void __init check_timer(void)
 	if (!global_clock_event)
 		return;
 
-	local_irq_save(flags);
+	local_irq_save_full(flags);
 
 	/*
 	 * get/set the timer IRQ vector:
@@ -2203,7 +2250,7 @@ static inline void __init check_timer(void)
 			goto out;
 		}
 		panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC");
-		local_irq_disable();
+		local_irq_disable_full();
 		clear_IO_APIC_pin(apic1, pin1);
 		if (!no_pin1)
 			apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
@@ -2227,7 +2274,7 @@ static inline void __init check_timer(void)
 		/*
 		 * Cleanup, just in case ...
 		 */
-		local_irq_disable();
+		local_irq_disable_full();
 		legacy_pic->mask(0);
 		clear_IO_APIC_pin(apic2, pin2);
 		apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
@@ -2244,7 +2291,7 @@ static inline void __init check_timer(void)
 		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
 		goto out;
 	}
-	local_irq_disable();
+	local_irq_disable_full();
 	legacy_pic->mask(0);
 	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
 	apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
@@ -2263,7 +2310,7 @@ static inline void __init check_timer(void)
 		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
 		goto out;
 	}
-	local_irq_disable();
+	local_irq_disable_full();
 	apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
 	if (apic_is_x2apic_enabled())
 		apic_printk(APIC_QUIET, KERN_INFO
@@ -2272,7 +2319,7 @@ static inline void __init check_timer(void)
 	panic("IO-APIC + timer doesn't work!  Boot with apic=debug and send a "
 		"report.  Then try booting with the 'noapic' option.\n");
 out:
-	local_irq_restore(flags);
+	local_irq_restore_full(flags);
 }
 
 /*
@@ -3018,13 +3065,13 @@ int mp_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
 	cfg = irqd_cfg(irq_data);
 	add_pin_to_irq_node(data, ioapic_alloc_attr_node(info), ioapic, pin);
 
-	local_irq_save(flags);
+	local_irq_save_full(flags);
 	if (info->ioapic.entry)
 		mp_setup_entry(cfg, data, info->ioapic.entry);
 	mp_register_handler(virq, data->trigger);
 	if (virq < nr_legacy_irqs())
 		legacy_pic->mask(virq);
-	local_irq_restore(flags);
+	local_irq_restore_full(flags);
 
 	apic_printk(APIC_VERBOSE, KERN_DEBUG
 		    "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> IRQ %d Mode:%i Active:%i Dest:%d)\n",
diff --git a/kernel/arch/x86/kernel/apic/ipi.c b/kernel/arch/x86/kernel/apic/ipi.c
index 387154e39e..bd2ffaef46 100644
--- a/kernel/arch/x86/kernel/apic/ipi.c
+++ b/kernel/arch/x86/kernel/apic/ipi.c
@@ -117,8 +117,10 @@ void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
 	 * cli/sti.  Otherwise we use an even cheaper single atomic write
 	 * to the APIC.
 	 */
+	unsigned long flags;
 	unsigned int cfg;
 
+	flags = hard_cond_local_irq_save();
 	/*
 	 * Wait for idle.
 	 */
@@ -137,6 +139,8 @@ void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
 	 * Send the IPI. The write to APIC_ICR fires this off.
 	 */
 	native_apic_mem_write(APIC_ICR, cfg);
+
+	hard_cond_local_irq_restore(flags);
 }
 
 /*
@@ -145,8 +149,10 @@ void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
  */
 void __default_send_IPI_dest_field(unsigned int mask, int vector, unsigned int dest)
 {
+	unsigned long flags;
 	unsigned long cfg;
 
+	flags = hard_cond_local_irq_save();
 	/*
 	 * Wait for idle.
 	 */
@@ -170,16 +176,18 @@ void __default_send_IPI_dest_field(unsigned int mask, int vector, unsigned int d
 	 * Send the IPI. The write to APIC_ICR fires this off.
 	 */
 	native_apic_mem_write(APIC_ICR, cfg);
+
+	hard_cond_local_irq_restore(flags);
 }
 
 void default_send_IPI_single_phys(int cpu, int vector)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu),
 				      vector, APIC_DEST_PHYSICAL);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
@@ -192,12 +200,12 @@ void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
 	 * to an arbitrary mask, so I do a unicast to each CPU instead.
 	 * - mbligh
 	 */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	for_each_cpu(query_cpu, mask) {
 		__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
 				query_cpu), vector, APIC_DEST_PHYSICAL);
 	}
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
@@ -209,14 +217,14 @@ void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
 
 	/* See Hack comment above */
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	for_each_cpu(query_cpu, mask) {
 		if (query_cpu == this_cpu)
 			continue;
 		__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
 				 query_cpu), vector, APIC_DEST_PHYSICAL);
 	}
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /*
@@ -256,12 +264,12 @@ void default_send_IPI_mask_sequence_logical(const struct cpumask *mask,
 	 * should be modified to do 1 message per cluster ID - mbligh
 	 */
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	for_each_cpu(query_cpu, mask)
 		__default_send_IPI_dest_field(
 			early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
 			vector, apic->dest_logical);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
@@ -273,7 +281,7 @@ void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
 
 	/* See Hack comment above */
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	for_each_cpu(query_cpu, mask) {
 		if (query_cpu == this_cpu)
 			continue;
@@ -281,7 +289,7 @@ void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
 			early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
 			vector, apic->dest_logical);
 		}
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /*
@@ -295,10 +303,10 @@ void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
 	if (!mask)
 		return;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
 	__default_send_IPI_dest_field(mask, vector, apic->dest_logical);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /* must come after the send_IPI functions above for inlining */
diff --git a/kernel/arch/x86/kernel/apic/msi.c b/kernel/arch/x86/kernel/apic/msi.c
index 6bd98a20fc..69dc36e63a 100644
--- a/kernel/arch/x86/kernel/apic/msi.c
+++ b/kernel/arch/x86/kernel/apic/msi.c
@@ -181,7 +181,8 @@ static struct irq_chip pci_msi_controller = {
 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
 	.irq_set_affinity	= msi_set_affinity,
 	.flags			= IRQCHIP_SKIP_SET_WAKE |
-				  IRQCHIP_AFFINITY_PRE_STARTUP,
+				  IRQCHIP_AFFINITY_PRE_STARTUP |
+				  IRQCHIP_PIPELINE_SAFE,
 };
 
 int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec,
@@ -251,7 +252,8 @@ static struct irq_chip pci_msi_ir_controller = {
 	.irq_ack		= irq_chip_ack_parent,
 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
 	.flags			= IRQCHIP_SKIP_SET_WAKE |
-				  IRQCHIP_AFFINITY_PRE_STARTUP,
+				  IRQCHIP_AFFINITY_PRE_STARTUP |
+				  IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct msi_domain_info pci_msi_ir_domain_info = {
@@ -294,7 +296,8 @@ static struct irq_chip dmar_msi_controller = {
 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
 	.irq_write_msi_msg	= dmar_msi_write_msg,
 	.flags			= IRQCHIP_SKIP_SET_WAKE |
-				  IRQCHIP_AFFINITY_PRE_STARTUP,
+				  IRQCHIP_AFFINITY_PRE_STARTUP |
+				  IRQCHIP_PIPELINE_SAFE,
 };
 
 static int dmar_msi_init(struct irq_domain *domain,
@@ -386,7 +389,8 @@ static struct irq_chip hpet_msi_controller __ro_after_init = {
 	.irq_set_affinity = msi_domain_set_affinity,
 	.irq_retrigger = irq_chip_retrigger_hierarchy,
 	.irq_write_msi_msg = hpet_msi_write_msg,
-	.flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_AFFINITY_PRE_STARTUP,
+	.flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_AFFINITY_PRE_STARTUP |
+		 IRQCHIP_PIPELINE_SAFE,
 };
 
 static int hpet_msi_init(struct irq_domain *domain,
diff --git a/kernel/arch/x86/kernel/apic/vector.c b/kernel/arch/x86/kernel/apic/vector.c
index bd557e9f5d..1cb7ab408b 100644
--- a/kernel/arch/x86/kernel/apic/vector.c
+++ b/kernel/arch/x86/kernel/apic/vector.c
@@ -39,7 +39,7 @@ struct apic_chip_data {
 
 struct irq_domain *x86_vector_domain;
 EXPORT_SYMBOL_GPL(x86_vector_domain);
-static DEFINE_RAW_SPINLOCK(vector_lock);
+static DEFINE_HARD_SPINLOCK(vector_lock);
 static cpumask_var_t vector_searchmask;
 static struct irq_chip lapic_controller;
 static struct irq_matrix *vector_matrix;
@@ -757,6 +757,10 @@ static struct irq_desc *__setup_vector_irq(int vector)
 {
 	int isairq = vector - ISA_IRQ_VECTOR(0);
 
+	/* Copy the cleanup vector if irqs are pipelined. */
+	if (IS_ENABLED(CONFIG_IRQ_PIPELINE) &&
+		vector == IRQ_MOVE_CLEANUP_VECTOR)
+		return irq_to_desc(IRQ_MOVE_CLEANUP_VECTOR); /* 1:1 mapping */
 	/* Check whether the irq is in the legacy space */
 	if (isairq < 0 || isairq >= nr_legacy_irqs())
 		return VECTOR_UNUSED;
@@ -791,9 +795,11 @@ void lapic_online(void)
 
 void lapic_offline(void)
 {
-	lock_vector_lock();
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&vector_lock, flags);
 	irq_matrix_offline(vector_matrix);
-	unlock_vector_lock();
+	raw_spin_unlock_irqrestore(&vector_lock, flags);
 }
 
 static int apic_set_affinity(struct irq_data *irqd,
@@ -801,6 +807,8 @@ static int apic_set_affinity(struct irq_data *irqd,
 {
 	int err;
 
+	WARN_ON_ONCE(irqs_pipelined() && !hard_irqs_disabled());
+
 	if (WARN_ON_ONCE(!irqd_is_activated(irqd)))
 		return -EIO;
 
@@ -830,10 +838,44 @@ static int apic_retrigger_irq(struct irq_data *irqd)
 	return 1;
 }
 
-void apic_ack_irq(struct irq_data *irqd)
+#if defined(CONFIG_IRQ_PIPELINE) &&	\
+	defined(CONFIG_GENERIC_PENDING_IRQ)
+
+static void apic_deferred_irq_move(struct irq_work *work)
+{
+	struct irq_data *irqd;
+	struct irq_desc *desc;
+	unsigned long flags;
+
+	irqd = container_of(work, struct irq_data, move_work);
+	desc = irq_data_to_desc(irqd);
+	raw_spin_lock_irqsave(&desc->lock, flags);
+	__irq_move_irq(irqd);
+	raw_spin_unlock_irqrestore(&desc->lock, flags);
+}
+
+static inline void apic_move_irq(struct irq_data *irqd)
+{
+	if (irqd_is_setaffinity_pending(irqd) &&
+		!irqd_is_setaffinity_blocked(irqd)) {
+		init_irq_work(&irqd->move_work, apic_deferred_irq_move);
+		irq_work_queue(&irqd->move_work);
+	}
+}
+
+#else
+
+static inline void apic_move_irq(struct irq_data *irqd)
 {
 	irq_move_irq(irqd);
-	ack_APIC_irq();
+}
+
+#endif
+
+void apic_ack_irq(struct irq_data *irqd)
+{
+	apic_move_irq(irqd);
+	__ack_APIC_irq();
 }
 
 void apic_ack_edge(struct irq_data *irqd)
@@ -876,15 +918,17 @@ static void free_moved_vector(struct apic_chip_data *apicd)
 	apicd->move_in_progress = 0;
 }
 
-DEFINE_IDTENTRY_SYSVEC(sysvec_irq_move_cleanup)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(IRQ_MOVE_CLEANUP_VECTOR,
+				 sysvec_irq_move_cleanup)
 {
 	struct hlist_head *clhead = this_cpu_ptr(&cleanup_list);
 	struct apic_chip_data *apicd;
 	struct hlist_node *tmp;
+	unsigned long flags;
 
 	ack_APIC_irq();
 	/* Prevent vectors vanishing under us */
-	raw_spin_lock(&vector_lock);
+	raw_spin_lock_irqsave(&vector_lock, flags);
 
 	hlist_for_each_entry_safe(apicd, tmp, clhead, clist) {
 		unsigned int irr, vector = apicd->prev_vector;
@@ -906,14 +950,15 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_irq_move_cleanup)
 		free_moved_vector(apicd);
 	}
 
-	raw_spin_unlock(&vector_lock);
+	raw_spin_unlock_irqrestore(&vector_lock, flags);
 }
 
 static void __send_cleanup_vector(struct apic_chip_data *apicd)
 {
+	unsigned long flags;
 	unsigned int cpu;
 
-	raw_spin_lock(&vector_lock);
+	raw_spin_lock_irqsave(&vector_lock, flags);
 	apicd->move_in_progress = 0;
 	cpu = apicd->prev_cpu;
 	if (cpu_online(cpu)) {
@@ -922,7 +967,7 @@ static void __send_cleanup_vector(struct apic_chip_data *apicd)
 	} else {
 		apicd->prev_vector = 0;
 	}
-	raw_spin_unlock(&vector_lock);
+	raw_spin_unlock_irqrestore(&vector_lock, flags);
 }
 
 void send_cleanup_vector(struct irq_cfg *cfg)
@@ -961,6 +1006,8 @@ void irq_force_complete_move(struct irq_desc *desc)
 	struct irq_data *irqd;
 	unsigned int vector;
 
+	WARN_ON_ONCE(irqs_pipelined() && !hard_irqs_disabled());
+
 	/*
 	 * The function is called for all descriptors regardless of which
 	 * irqdomain they belong to. For example if an IRQ is provided by
@@ -1051,9 +1098,10 @@ void irq_force_complete_move(struct irq_desc *desc)
 int lapic_can_unplug_cpu(void)
 {
 	unsigned int rsvd, avl, tomove, cpu = smp_processor_id();
+	unsigned long flags;
 	int ret = 0;
 
-	raw_spin_lock(&vector_lock);
+	raw_spin_lock_irqsave(&vector_lock, flags);
 	tomove = irq_matrix_allocated(vector_matrix);
 	avl = irq_matrix_available(vector_matrix, true);
 	if (avl < tomove) {
@@ -1068,7 +1116,7 @@ int lapic_can_unplug_cpu(void)
 			rsvd, avl);
 	}
 out:
-	raw_spin_unlock(&vector_lock);
+	raw_spin_unlock_irqrestore(&vector_lock, flags);
 	return ret;
 }
 #endif /* HOTPLUG_CPU */
diff --git a/kernel/arch/x86/kernel/apic/x2apic_cluster.c b/kernel/arch/x86/kernel/apic/x2apic_cluster.c
index 7eec3c154f..52fdf80e4d 100644
--- a/kernel/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/kernel/arch/x86/kernel/apic/x2apic_cluster.c
@@ -44,7 +44,7 @@ __x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
 
 	/* x2apic MSRs are special and need a special fence: */
 	weak_wrmsr_fence();
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	tmpmsk = this_cpu_cpumask_var_ptr(ipi_mask);
 	cpumask_copy(tmpmsk, mask);
@@ -68,7 +68,7 @@ __x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
 		cpumask_andnot(tmpmsk, tmpmsk, &cmsk->mask);
 	}
 
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
diff --git a/kernel/arch/x86/kernel/apic/x2apic_phys.c b/kernel/arch/x86/kernel/apic/x2apic_phys.c
index 032a00e5d9..72ebc33401 100644
--- a/kernel/arch/x86/kernel/apic/x2apic_phys.c
+++ b/kernel/arch/x86/kernel/apic/x2apic_phys.c
@@ -58,7 +58,7 @@ __x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
 	/* x2apic MSRs are special and need a special fence: */
 	weak_wrmsr_fence();
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	this_cpu = smp_processor_id();
 	for_each_cpu(query_cpu, mask) {
@@ -67,7 +67,7 @@ __x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
 		__x2apic_send_IPI_dest(per_cpu(x86_cpu_to_apicid, query_cpu),
 				       vector, APIC_DEST_PHYSICAL);
 	}
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
diff --git a/kernel/arch/x86/kernel/asm-offsets.c b/kernel/arch/x86/kernel/asm-offsets.c
index 70b7154f4b..09c539ff49 100644
--- a/kernel/arch/x86/kernel/asm-offsets.c
+++ b/kernel/arch/x86/kernel/asm-offsets.c
@@ -38,6 +38,9 @@ static void __used common(void)
 #endif
 
 	BLANK();
+#ifdef CONFIG_IRQ_PIPELINE
+	DEFINE(OOB_stage_mask, STAGE_MASK);
+#endif
 	OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
 
 	BLANK();
diff --git a/kernel/arch/x86/kernel/cpu/acrn.c b/kernel/arch/x86/kernel/cpu/acrn.c
index 0b2c03943a..7f0694b812 100644
--- a/kernel/arch/x86/kernel/cpu/acrn.c
+++ b/kernel/arch/x86/kernel/cpu/acrn.c
@@ -35,7 +35,8 @@ static bool acrn_x2apic_available(void)
 
 static void (*acrn_intr_handler)(void);
 
-DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(HYPERVISOR_CALLBACK_VECTOR,
+				 sysvec_acrn_hv_callback)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
diff --git a/kernel/arch/x86/kernel/cpu/mce/amd.c b/kernel/arch/x86/kernel/cpu/mce/amd.c
index 09f7c65234..d158d8328a 100644
--- a/kernel/arch/x86/kernel/cpu/mce/amd.c
+++ b/kernel/arch/x86/kernel/cpu/mce/amd.c
@@ -921,13 +921,18 @@ static void __log_error(unsigned int bank, u64 status, u64 addr, u64 misc)
 	mce_log(&m);
 }
 
+/*
+ * irq_pipeline: Deferred error events have NMI semantics wrt to
+ * pipelining, they can and should be handled immediately out of the
+ * IDT.
+ */
 DEFINE_IDTENTRY_SYSVEC(sysvec_deferred_error)
 {
 	trace_deferred_error_apic_entry(DEFERRED_ERROR_VECTOR);
 	inc_irq_stat(irq_deferred_error_count);
 	deferred_error_int_vector();
 	trace_deferred_error_apic_exit(DEFERRED_ERROR_VECTOR);
-	ack_APIC_irq();
+	__ack_APIC_irq();
 }
 
 /*
diff --git a/kernel/arch/x86/kernel/cpu/mce/core.c b/kernel/arch/x86/kernel/cpu/mce/core.c
index 5cf1a02440..e18dae3db6 100644
--- a/kernel/arch/x86/kernel/cpu/mce/core.c
+++ b/kernel/arch/x86/kernel/cpu/mce/core.c
@@ -1473,7 +1473,9 @@ noinstr void do_machine_check(struct pt_regs *regs)
 		/* If this triggers there is no way to recover. Die hard. */
 		BUG_ON(!on_thread_stack() || !user_mode(regs));
 
+		hard_local_irq_enable();
 		queue_task_work(&m, msg, kill_it);
+		hard_local_irq_disable();
 
 	} else {
 		/*
diff --git a/kernel/arch/x86/kernel/cpu/mce/therm_throt.c b/kernel/arch/x86/kernel/cpu/mce/therm_throt.c
index a7cd2d203c..115dd0b146 100644
--- a/kernel/arch/x86/kernel/cpu/mce/therm_throt.c
+++ b/kernel/arch/x86/kernel/cpu/mce/therm_throt.c
@@ -614,13 +614,17 @@ static void unexpected_thermal_interrupt(void)
 
 static void (*smp_thermal_vector)(void) = unexpected_thermal_interrupt;
 
+/*
+ * irq_pipeline: MCE have NMI semantics wrt to pipelining, they can
+ * and should be handled immediately out of the IDT.
+ */
 DEFINE_IDTENTRY_SYSVEC(sysvec_thermal)
 {
 	trace_thermal_apic_entry(THERMAL_APIC_VECTOR);
 	inc_irq_stat(irq_thermal_count);
 	smp_thermal_vector();
 	trace_thermal_apic_exit(THERMAL_APIC_VECTOR);
-	ack_APIC_irq();
+	__ack_APIC_irq();
 }
 
 /* Thermal monitoring depends on APIC, ACPI and clock modulation */
diff --git a/kernel/arch/x86/kernel/cpu/mce/threshold.c b/kernel/arch/x86/kernel/cpu/mce/threshold.c
index 6a059a0350..a2515dc6b5 100644
--- a/kernel/arch/x86/kernel/cpu/mce/threshold.c
+++ b/kernel/arch/x86/kernel/cpu/mce/threshold.c
@@ -27,5 +27,5 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_threshold)
 	inc_irq_stat(irq_threshold_count);
 	mce_threshold_vector();
 	trace_threshold_apic_exit(THRESHOLD_APIC_VECTOR);
-	ack_APIC_irq();
+	__ack_APIC_irq();
 }
diff --git a/kernel/arch/x86/kernel/cpu/mshyperv.c b/kernel/arch/x86/kernel/cpu/mshyperv.c
index 021cd06773..ffa7ff0bd1 100644
--- a/kernel/arch/x86/kernel/cpu/mshyperv.c
+++ b/kernel/arch/x86/kernel/cpu/mshyperv.c
@@ -41,7 +41,8 @@ static void (*hv_stimer0_handler)(void);
 static void (*hv_kexec_handler)(void);
 static void (*hv_crash_handler)(struct pt_regs *regs);
 
-DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_callback)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(HYPERVISOR_CALLBACK_VECTOR,
+				 sysvec_hyperv_callback)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
@@ -77,7 +78,8 @@ EXPORT_SYMBOL_GPL(hv_remove_vmbus_irq);
  * Routines to do per-architecture handling of stimer0
  * interrupts when in Direct Mode
  */
-DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(HYPERV_STIMER0_VECTOR,
+				 sysvec_hyperv_stimer0)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
diff --git a/kernel/arch/x86/kernel/cpu/mtrr/generic.c b/kernel/arch/x86/kernel/cpu/mtrr/generic.c
index a29997e6cf..fc15869244 100644
--- a/kernel/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/kernel/arch/x86/kernel/cpu/mtrr/generic.c
@@ -450,13 +450,13 @@ void __init mtrr_bp_pat_init(void)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	prepare_set();
 
 	pat_init();
 
 	post_set();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /* Grab all of the MTRR state for this CPU into *state */
@@ -797,7 +797,7 @@ static void generic_set_all(void)
 	unsigned long mask, count;
 	unsigned long flags;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	prepare_set();
 
 	/* Actually set the state */
@@ -807,7 +807,7 @@ static void generic_set_all(void)
 	pat_init();
 
 	post_set();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	/* Use the atomic bitops to update the global mask */
 	for (count = 0; count < sizeof(mask) * 8; ++count) {
@@ -836,7 +836,7 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
 
 	vr = &mtrr_state.var_ranges[reg];
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	prepare_set();
 
 	if (size == 0) {
@@ -857,7 +857,7 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
 	}
 
 	post_set();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 int generic_validate_add_page(unsigned long base, unsigned long size,
diff --git a/kernel/arch/x86/kernel/dumpstack.c b/kernel/arch/x86/kernel/dumpstack.c
index 97aa900386..0fe58edc65 100644
--- a/kernel/arch/x86/kernel/dumpstack.c
+++ b/kernel/arch/x86/kernel/dumpstack.c
@@ -7,6 +7,7 @@
 #include <linux/uaccess.h>
 #include <linux/utsname.h>
 #include <linux/hardirq.h>
+#include <linux/irq_pipeline.h>
 #include <linux/kdebug.h>
 #include <linux/module.h>
 #include <linux/ptrace.h>
@@ -335,7 +336,7 @@ unsigned long oops_begin(void)
 	oops_enter();
 
 	/* racy, but better than risking deadlock. */
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	cpu = smp_processor_id();
 	if (!arch_spin_trylock(&die_lock)) {
 		if (cpu == die_owner)
@@ -365,7 +366,7 @@ void oops_end(unsigned long flags, struct pt_regs *regs, int signr)
 	if (!die_nest_count)
 		/* Nest count reaches zero, release the lock. */
 		arch_spin_unlock(&die_lock);
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	oops_exit();
 
 	/* Executive summary in case the oops scrolled away */
@@ -394,6 +395,8 @@ static void __die_header(const char *str, struct pt_regs *regs, long err)
 {
 	const char *pr = "";
 
+	irq_pipeline_oops();
+
 	/* Save the regs of the first oops for the executive summary later. */
 	if (!die_counter)
 		exec_summary_regs = *regs;
@@ -402,13 +405,14 @@ static void __die_header(const char *str, struct pt_regs *regs, long err)
 		pr = IS_ENABLED(CONFIG_PREEMPT_RT) ? " PREEMPT_RT" : " PREEMPT";
 
 	printk(KERN_DEFAULT
-	       "%s: %04lx [#%d]%s%s%s%s%s\n", str, err & 0xffff, ++die_counter,
+	       "%s: %04lx [#%d]%s%s%s%s%s%s\n", str, err & 0xffff, ++die_counter,
 	       pr,
 	       IS_ENABLED(CONFIG_SMP)     ? " SMP"             : "",
 	       debug_pagealloc_enabled()  ? " DEBUG_PAGEALLOC" : "",
 	       IS_ENABLED(CONFIG_KASAN)   ? " KASAN"           : "",
 	       IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION) ?
-	       (boot_cpu_has(X86_FEATURE_PTI) ? " PTI" : " NOPTI") : "");
+	       (boot_cpu_has(X86_FEATURE_PTI) ? " PTI" : " NOPTI") : "",
+	       irqs_pipelined()           ? " IRQ_PIPELINE"    : "");
 }
 NOKPROBE_SYMBOL(__die_header);
 
diff --git a/kernel/arch/x86/kernel/fpu/core.c b/kernel/arch/x86/kernel/fpu/core.c
index 571220ac8b..49e2853a24 100644
--- a/kernel/arch/x86/kernel/fpu/core.c
+++ b/kernel/arch/x86/kernel/fpu/core.c
@@ -15,6 +15,7 @@
 
 #include <linux/hardirq.h>
 #include <linux/pkeys.h>
+#include <linux/cpuhotplug.h>
 
 #define CREATE_TRACE_POINTS
 #include <asm/trace/fpu.h>
@@ -76,9 +77,10 @@ static bool interrupted_user_mode(void)
  */
 bool irq_fpu_usable(void)
 {
-	return !in_interrupt() ||
-		interrupted_user_mode() ||
-		interrupted_kernel_fpu_idle();
+	return running_inband() &&
+		(!in_interrupt() ||
+			interrupted_user_mode() ||
+			interrupted_kernel_fpu_idle());
 }
 EXPORT_SYMBOL(irq_fpu_usable);
 
@@ -123,11 +125,15 @@ EXPORT_SYMBOL(copy_fpregs_to_fpstate);
 
 void kernel_fpu_begin_mask(unsigned int kfpu_mask)
 {
+	unsigned long flags;
+
 	preempt_disable();
 
 	WARN_ON_FPU(!irq_fpu_usable());
 	WARN_ON_FPU(this_cpu_read(in_kernel_fpu));
 
+	flags = hard_cond_local_irq_save();
+
 	this_cpu_write(in_kernel_fpu, true);
 
 	if (!(current->flags & PF_KTHREAD) &&
@@ -139,6 +145,7 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask)
 		 */
 		copy_fpregs_to_fpstate(&current->thread.fpu);
 	}
+
 	__cpu_invalidate_fpregs_state();
 
 	/* Put sane initial values into the control registers. */
@@ -147,6 +154,8 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask)
 
 	if (unlikely(kfpu_mask & KFPU_387) && boot_cpu_has(X86_FEATURE_FPU))
 		asm volatile ("fninit");
+
+	hard_cond_local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(kernel_fpu_begin_mask);
 
@@ -166,9 +175,11 @@ EXPORT_SYMBOL_GPL(kernel_fpu_end);
  */
 void fpu__save(struct fpu *fpu)
 {
+	unsigned long flags;
+
 	WARN_ON_FPU(fpu != &current->thread.fpu);
 
-	fpregs_lock();
+	flags = fpregs_lock();
 	trace_x86_fpu_before_save(fpu);
 
 	if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
@@ -178,7 +189,7 @@ void fpu__save(struct fpu *fpu)
 	}
 
 	trace_x86_fpu_after_save(fpu);
-	fpregs_unlock();
+	fpregs_unlock(flags);
 }
 
 /*
@@ -214,6 +225,7 @@ int fpu__copy(struct task_struct *dst, struct task_struct *src)
 {
 	struct fpu *dst_fpu = &dst->thread.fpu;
 	struct fpu *src_fpu = &src->thread.fpu;
+	unsigned long flags;
 
 	dst_fpu->last_cpu = -1;
 
@@ -236,14 +248,14 @@ int fpu__copy(struct task_struct *dst, struct task_struct *src)
 	 * ( The function 'fails' in the FNSAVE case, which destroys
 	 *   register contents so we have to load them back. )
 	 */
-	fpregs_lock();
+	flags = fpregs_lock();
 	if (test_thread_flag(TIF_NEED_FPU_LOAD))
 		memcpy(&dst_fpu->state, &src_fpu->state, fpu_kernel_xstate_size);
 
 	else if (!copy_fpregs_to_fpstate(dst_fpu))
 		copy_kernel_to_fpregs(&dst_fpu->state);
 
-	fpregs_unlock();
+	fpregs_unlock(flags);
 
 	set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD);
 
@@ -321,7 +333,9 @@ void fpu__prepare_write(struct fpu *fpu)
  */
 void fpu__drop(struct fpu *fpu)
 {
-	preempt_disable();
+	unsigned long flags;
+
+	flags = hard_preempt_disable();
 
 	if (fpu == &current->thread.fpu) {
 		/* Ignore delayed exceptions from user space */
@@ -333,7 +347,7 @@ void fpu__drop(struct fpu *fpu)
 
 	trace_x86_fpu_dropped(fpu);
 
-	preempt_enable();
+	hard_preempt_enable(flags);
 }
 
 /*
@@ -361,15 +375,19 @@ static inline void copy_init_fpstate_to_fpregs(u64 features_mask)
  */
 static void fpu__clear(struct fpu *fpu, bool user_only)
 {
+	unsigned long flags;
+
 	WARN_ON_FPU(fpu != &current->thread.fpu);
 
 	if (!static_cpu_has(X86_FEATURE_FPU)) {
+		flags = hard_cond_local_irq_save();
 		fpu__drop(fpu);
 		fpu__initialize(fpu);
+		hard_cond_local_irq_restore(flags);
 		return;
 	}
 
-	fpregs_lock();
+	flags = fpregs_lock();
 
 	if (user_only) {
 		if (!fpregs_state_valid(fpu, smp_processor_id()) &&
@@ -382,7 +400,7 @@ static void fpu__clear(struct fpu *fpu, bool user_only)
 	}
 
 	fpregs_mark_activate();
-	fpregs_unlock();
+	fpregs_unlock(flags);
 }
 
 void fpu__clear_user_states(struct fpu *fpu)
@@ -400,10 +418,14 @@ void fpu__clear_all(struct fpu *fpu)
  */
 void switch_fpu_return(void)
 {
+	unsigned long flags;
+
 	if (!static_cpu_has(X86_FEATURE_FPU))
 		return;
 
+	flags = hard_cond_local_irq_save();
 	__fpregs_load_activate();
+	hard_cond_local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(switch_fpu_return);
 
@@ -503,3 +525,70 @@ int fpu__exception_code(struct fpu *fpu, int trap_nr)
 	 */
 	return 0;
 }
+
+#ifdef CONFIG_DOVETAIL
+
+/*
+ * Holds the in-kernel fpu state when preempted by a task running on
+ * the out-of-band stage.
+ */
+static DEFINE_PER_CPU(struct fpu *, in_kernel_fpstate);
+
+static int fpu__init_kernel_fpstate(unsigned int cpu)
+{
+	struct fpu *fpu;
+
+	fpu = kzalloc(sizeof(*fpu) + fpu_kernel_xstate_size, GFP_KERNEL);
+	if (fpu == NULL)
+		return -ENOMEM;
+
+	this_cpu_write(in_kernel_fpstate, fpu);
+	fpstate_init(&fpu->state);
+
+	return 0;
+}
+
+static int fpu__drop_kernel_fpstate(unsigned int cpu)
+{
+	struct fpu *fpu = this_cpu_read(in_kernel_fpstate);
+
+	kfree(fpu);
+
+	return 0;
+}
+
+void fpu__suspend_inband(void)
+{
+	struct fpu *kfpu = this_cpu_read(in_kernel_fpstate);
+	struct task_struct *tsk = current;
+
+	if (kernel_fpu_disabled()) {
+		copy_fpregs_to_fpstate(kfpu);
+		__cpu_invalidate_fpregs_state();
+		oob_fpu_set_preempt(&tsk->thread.fpu);
+	}
+}
+
+void fpu__resume_inband(void)
+{
+	struct fpu *kfpu = this_cpu_read(in_kernel_fpstate);
+	struct task_struct *tsk = current;
+
+	if (oob_fpu_preempted(&tsk->thread.fpu)) {
+		copy_kernel_to_fpregs(&kfpu->state);
+		__cpu_invalidate_fpregs_state();
+		oob_fpu_clear_preempt(&tsk->thread.fpu);
+	} else if (!(tsk->flags & PF_KTHREAD) &&
+		test_thread_flag(TIF_NEED_FPU_LOAD))
+		switch_fpu_return();
+}
+
+static void __init fpu__init_dovetail(void)
+{
+	cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
+			"platform/x86/dovetail:online",
+			fpu__init_kernel_fpstate, fpu__drop_kernel_fpstate);
+}
+core_initcall(fpu__init_dovetail);
+
+#endif
diff --git a/kernel/arch/x86/kernel/fpu/signal.c b/kernel/arch/x86/kernel/fpu/signal.c
index b7b92cdf3a..20d04a39b9 100644
--- a/kernel/arch/x86/kernel/fpu/signal.c
+++ b/kernel/arch/x86/kernel/fpu/signal.c
@@ -61,11 +61,12 @@ static inline int save_fsave_header(struct task_struct *tsk, void __user *buf)
 		struct xregs_state *xsave = &tsk->thread.fpu.state.xsave;
 		struct user_i387_ia32_struct env;
 		struct _fpstate_32 __user *fp = buf;
+		unsigned long flags;
 
-		fpregs_lock();
+		flags = fpregs_lock();
 		if (!test_thread_flag(TIF_NEED_FPU_LOAD))
 			copy_fxregs_to_kernel(&tsk->thread.fpu);
-		fpregs_unlock();
+		fpregs_unlock(flags);
 
 		convert_from_fxsr(&env, tsk);
 
@@ -165,6 +166,7 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
 {
 	struct task_struct *tsk = current;
 	int ia32_fxstate = (buf != buf_fx);
+	unsigned long flags;
 	int ret;
 
 	ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
@@ -186,14 +188,14 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size)
 	 * userland's stack frame which will likely succeed. If it does not,
 	 * resolve the fault in the user memory and try again.
 	 */
-	fpregs_lock();
+	flags = fpregs_lock();
 	if (test_thread_flag(TIF_NEED_FPU_LOAD))
 		__fpregs_load_activate();
 
 	pagefault_disable();
 	ret = copy_fpregs_to_sigframe(buf_fx);
 	pagefault_enable();
-	fpregs_unlock();
+	fpregs_unlock(flags);
 
 	if (ret) {
 		if (!fault_in_pages_writeable(buf_fx, fpu_user_xstate_size))
@@ -286,6 +288,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 	struct fpu *fpu = &tsk->thread.fpu;
 	struct user_i387_ia32_struct env;
 	u64 user_xfeatures = 0;
+	unsigned long flags;
 	int fx_only = 0;
 	int ret = 0;
 
@@ -337,7 +340,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 		 * going through the kernel buffer with the enabled pagefault
 		 * handler.
 		 */
-		fpregs_lock();
+		flags = fpregs_lock();
 		pagefault_disable();
 		ret = copy_user_to_fpregs_zeroing(buf_fx, user_xfeatures, fx_only);
 		pagefault_enable();
@@ -360,7 +363,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 				copy_kernel_to_xregs(&fpu->state.xsave,
 						     xfeatures_mask_supervisor());
 			fpregs_mark_activate();
-			fpregs_unlock();
+			fpregs_unlock(flags);
 			return 0;
 		}
 
@@ -382,7 +385,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 		if (test_thread_flag(TIF_NEED_FPU_LOAD))
 			__cpu_invalidate_fpregs_state();
 
-		fpregs_unlock();
+		fpregs_unlock(flags);
 	} else {
 		/*
 		 * For 32-bit frames with fxstate, copy the fxstate so it can
@@ -400,7 +403,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 	 * to be loaded again on return to userland (overriding last_cpu avoids
 	 * the optimisation).
 	 */
-	fpregs_lock();
+	flags = fpregs_lock();
 
 	if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
 
@@ -413,7 +416,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 		set_thread_flag(TIF_NEED_FPU_LOAD);
 	}
 	__fpu_invalidate_fpregs_state(fpu);
-	fpregs_unlock();
+	fpregs_unlock(flags);
 
 	if (use_xsave() && !fx_only) {
 		u64 init_bv = xfeatures_mask_user() & ~user_xfeatures;
@@ -425,7 +428,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 		sanitize_restored_user_xstate(&fpu->state, envp, user_xfeatures,
 					      fx_only);
 
-		fpregs_lock();
+		flags = fpregs_lock();
 		if (unlikely(init_bv))
 			copy_kernel_to_xregs(&init_fpstate.xsave, init_bv);
 
@@ -446,7 +449,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 		sanitize_restored_user_xstate(&fpu->state, envp, user_xfeatures,
 					      fx_only);
 
-		fpregs_lock();
+		flags = fpregs_lock();
 		if (use_xsave()) {
 			u64 init_bv;
 
@@ -460,14 +463,14 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
 		if (ret)
 			goto out;
 
-		fpregs_lock();
+		flags = fpregs_lock();
 		ret = copy_kernel_to_fregs_err(&fpu->state.fsave);
 	}
 	if (!ret)
 		fpregs_mark_activate();
 	else
 		fpregs_deactivate(fpu);
-	fpregs_unlock();
+	fpregs_unlock(flags);
 
 out:
 	if (ret)
diff --git a/kernel/arch/x86/kernel/hpet.c b/kernel/arch/x86/kernel/hpet.c
index 574df24a8e..f44da6ca8e 100644
--- a/kernel/arch/x86/kernel/hpet.c
+++ b/kernel/arch/x86/kernel/hpet.c
@@ -406,7 +406,7 @@ static void hpet_init_clockevent(struct hpet_channel *hc, unsigned int rating)
 	evt->set_next_event	= hpet_clkevt_set_next_event;
 	evt->set_state_shutdown	= hpet_clkevt_set_state_shutdown;
 
-	evt->features = CLOCK_EVT_FEAT_ONESHOT;
+	evt->features = CLOCK_EVT_FEAT_ONESHOT|CLOCK_EVT_FEAT_PIPELINE;
 	if (hc->boot_cfg & HPET_TN_PERIODIC) {
 		evt->features		|= CLOCK_EVT_FEAT_PERIODIC;
 		evt->set_state_periodic	= hpet_clkevt_set_state_periodic;
@@ -519,7 +519,7 @@ static irqreturn_t hpet_msi_interrupt_handler(int irq, void *data)
 		return IRQ_HANDLED;
 	}
 
-	evt->event_handler(evt);
+	clockevents_handle_event(evt);
 	return IRQ_HANDLED;
 }
 
@@ -702,7 +702,7 @@ static u64 read_hpet(struct clocksource *cs)
 	if (arch_spin_is_locked(&old.lock))
 		goto contended;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	if (arch_spin_trylock(&hpet.lock)) {
 		new.value = hpet_readl(HPET_COUNTER);
 		/*
@@ -710,10 +710,10 @@ static u64 read_hpet(struct clocksource *cs)
 		 */
 		WRITE_ONCE(hpet.value, new.value);
 		arch_spin_unlock(&hpet.lock);
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		return (u64)new.value;
 	}
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 contended:
 	/*
diff --git a/kernel/arch/x86/kernel/i8259.c b/kernel/arch/x86/kernel/i8259.c
index 282b4ee133..6abdbd0e37 100644
--- a/kernel/arch/x86/kernel/i8259.c
+++ b/kernel/arch/x86/kernel/i8259.c
@@ -33,7 +33,7 @@
 static void init_8259A(int auto_eoi);
 
 static int i8259A_auto_eoi;
-DEFINE_RAW_SPINLOCK(i8259A_lock);
+DEFINE_HARD_SPINLOCK(i8259A_lock);
 
 /*
  * 8259A PIC functions to handle ISA devices:
@@ -227,6 +227,7 @@ struct irq_chip i8259A_chip = {
 	.irq_disable	= disable_8259A_irq,
 	.irq_unmask	= enable_8259A_irq,
 	.irq_mask_ack	= mask_and_ack_8259A,
+	.flags		= IRQCHIP_PIPELINE_SAFE,
 };
 
 static char irq_trigger[2];
diff --git a/kernel/arch/x86/kernel/idt.c b/kernel/arch/x86/kernel/idt.c
index ee1a283f8e..fb4333d54d 100644
--- a/kernel/arch/x86/kernel/idt.c
+++ b/kernel/arch/x86/kernel/idt.c
@@ -117,6 +117,10 @@ static const __initconst struct idt_data apic_idts[] = {
 	INTG(CALL_FUNCTION_SINGLE_VECTOR,	asm_sysvec_call_function_single),
 	INTG(IRQ_MOVE_CLEANUP_VECTOR,		asm_sysvec_irq_move_cleanup),
 	INTG(REBOOT_VECTOR,			asm_sysvec_reboot),
+#ifdef CONFIG_IRQ_PIPELINE
+	INTG(RESCHEDULE_OOB_VECTOR,		asm_sysvec_reschedule_oob_ipi),
+	INTG(TIMER_OOB_VECTOR,			asm_sysvec_timer_oob_ipi),
+#endif
 #endif
 
 #ifdef CONFIG_X86_THERMAL_VECTOR
diff --git a/kernel/arch/x86/kernel/irq.c b/kernel/arch/x86/kernel/irq.c
index ce904c89c6..753e4d61c2 100644
--- a/kernel/arch/x86/kernel/irq.c
+++ b/kernel/arch/x86/kernel/irq.c
@@ -4,6 +4,7 @@
  */
 #include <linux/cpu.h>
 #include <linux/interrupt.h>
+#include <linux/irq_pipeline.h>
 #include <linux/kernel_stat.h>
 #include <linux/of.h>
 #include <linux/seq_file.h>
@@ -48,7 +49,7 @@ void ack_bad_irq(unsigned int irq)
 	 * completely.
 	 * But only ack when the APIC is enabled -AK
 	 */
-	ack_APIC_irq();
+	__ack_APIC_irq();
 }
 
 #define irq_stats(x)		(&per_cpu(irq_stat, x))
@@ -235,8 +236,11 @@ static __always_inline void handle_irq(struct irq_desc *desc,
 /*
  * common_interrupt() handles all normal device IRQ's (the special SMP
  * cross-CPU interrupts have their own entry points).
+ *
+ * Compiled out if CONFIG_IRQ_PIPELINE is enabled, replaced by
+ * arch_handle_irq().
  */
-DEFINE_IDTENTRY_IRQ(common_interrupt)
+DEFINE_IDTENTRY_IRQ_PIPELINED(common_interrupt)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 	struct irq_desc *desc;
@@ -268,7 +272,8 @@ void (*x86_platform_ipi_callback)(void) = NULL;
 /*
  * Handler for X86_PLATFORM_IPI_VECTOR.
  */
-DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(X86_PLATFORM_IPI_VECTOR,
+				 sysvec_x86_platform_ipi)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
@@ -300,7 +305,8 @@ EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
 /*
  * Handler for POSTED_INTERRUPT_VECTOR.
  */
-DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
+DEFINE_IDTENTRY_SYSVEC_SIMPLE_PIPELINED(POSTED_INTR_VECTOR,
+					sysvec_kvm_posted_intr_ipi)
 {
 	ack_APIC_irq();
 	inc_irq_stat(kvm_posted_intr_ipis);
@@ -309,7 +315,8 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
 /*
  * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
  */
-DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(POSTED_INTR_WAKEUP_VECTOR,
+				 sysvec_kvm_posted_intr_wakeup_ipi)
 {
 	ack_APIC_irq();
 	inc_irq_stat(kvm_posted_intr_wakeup_ipis);
@@ -319,7 +326,8 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
 /*
  * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
  */
-DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi)
+DEFINE_IDTENTRY_SYSVEC_SIMPLE_PIPELINED(POSTED_INTR_NESTED_VECTOR,
+					sysvec_kvm_posted_intr_nested_ipi)
 {
 	ack_APIC_irq();
 	inc_irq_stat(kvm_posted_intr_nested_ipis);
diff --git a/kernel/arch/x86/kernel/irq_pipeline.c b/kernel/arch/x86/kernel/irq_pipeline.c
new file mode 100644
index 0000000000..725a326c4e
--- /dev/null
+++ b/kernel/arch/x86/kernel/irq_pipeline.c
@@ -0,0 +1,387 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2019 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/irq.h>
+#include <linux/irq_pipeline.h>
+#include <asm/irqdomain.h>
+#include <asm/apic.h>
+#include <asm/traps.h>
+#include <asm/irq_work.h>
+#include <asm/mshyperv.h>
+#include <asm/idtentry.h>
+
+static struct irq_domain *sipic_domain;
+
+static void sipic_irq_noop(struct irq_data *data) { }
+
+static unsigned int sipic_irq_noop_ret(struct irq_data *data)
+{
+	return 0;
+}
+
+static struct irq_chip sipic_chip = {
+	.name		= "SIPIC",
+	.irq_startup	= sipic_irq_noop_ret,
+	.irq_shutdown	= sipic_irq_noop,
+	.irq_enable	= sipic_irq_noop,
+	.irq_disable	= sipic_irq_noop,
+	.flags		= IRQCHIP_PIPELINE_SAFE | IRQCHIP_SKIP_SET_WAKE,
+};
+
+void handle_apic_irq(struct irq_desc *desc)
+{
+	if (WARN_ON_ONCE(irq_pipeline_debug() && !on_pipeline_entry()))
+		return;
+
+	/*
+	 * MCE events are non-maskable therefore their in-band
+	 * handlers have to be oob-compatible by construction. Those
+	 * handlers run immediately out of the IDT for this reason as
+	 * well. We won't see them here since they are not routed via
+	 * arch_handle_irq() -> generic_pipeline_irq().
+	 *
+	 * All we need to do at this stage is to acknowledge other
+	 * APIC events, then pipeline the corresponding interrupt from
+	 * our synthetic controller chip (SIPIC).
+	 */
+	__ack_APIC_irq();
+
+	handle_oob_irq(desc);
+}
+
+void irq_send_oob_ipi(unsigned int ipi,
+		const struct cpumask *cpumask)
+{
+	apic->send_IPI_mask_allbutself(cpumask,	apicm_irq_vector(ipi));
+}
+EXPORT_SYMBOL_GPL(irq_send_oob_ipi);
+
+static void do_sysvec_inband(struct irq_desc *desc)
+{
+	unsigned int irq = irq_desc_get_irq(desc);
+	struct pt_regs *regs = get_irq_regs();
+	int vector = apicm_irq_vector(irq);
+
+	/*
+	 * This code only sees pipelined sysvec events tagged with
+	 * DEFINE_IDTENTRY_SYSVEC_PIPELINED:
+	 *
+	 * 	arch_handle_irq(irq)
+	 *		generic_pipeline_irq(irq)
+	 *			handle_apic_irq(irq)
+	 *				handle_oob_irq(irq)
+	 *				[...irq_post_inband...]
+	 *
+	 *      arch_do_IRQ_pipelined(desc)
+	 *      <switch_to_irqstack>
+	 *                |
+	 *                v
+	 *	do_sysvec_inband(desc)
+	 *
+	 * System vectors which are still tagged as
+	 * DEFINE_IDTENTRY_SYSVEC/DEFINE_IDTENTRY_SYSVEC_SIMPLE are
+	 * directly dispatched out of the IDT, assuming their handler
+	 * is oob-safe (like NMI handlers) therefore never reach this
+	 * in-band stage handler.
+	 */
+
+	switch (vector) {
+#ifdef CONFIG_SMP
+	case RESCHEDULE_VECTOR:
+		__sysvec_reschedule_ipi(regs);
+		break;
+	case CALL_FUNCTION_VECTOR:
+		__sysvec_call_function(regs);
+		break;
+	case CALL_FUNCTION_SINGLE_VECTOR:
+		__sysvec_call_function_single(regs);
+		break;
+	case REBOOT_VECTOR:
+		__sysvec_reboot(regs);
+		break;
+#endif
+	case X86_PLATFORM_IPI_VECTOR:
+		__sysvec_x86_platform_ipi(regs);
+		break;
+	case IRQ_WORK_VECTOR:
+		__sysvec_irq_work(regs);
+		break;
+#ifdef CONFIG_HAVE_KVM
+	case POSTED_INTR_VECTOR:
+		__sysvec_kvm_posted_intr_ipi(regs);
+		break;
+	case POSTED_INTR_WAKEUP_VECTOR:
+		__sysvec_kvm_posted_intr_wakeup_ipi(regs);
+		break;
+	case POSTED_INTR_NESTED_VECTOR:
+		__sysvec_kvm_posted_intr_nested_ipi(regs);
+		break;
+#endif
+#ifdef CONFIG_HYPERV
+	case HYPERVISOR_CALLBACK_VECTOR:
+		__sysvec_hyperv_callback(regs);
+		break;
+	case HYPERV_REENLIGHTENMENT_VECTOR:
+		__sysvec_hyperv_reenlightenment(regs);
+		break;
+	case HYPERV_STIMER0_VECTOR:
+		__sysvec_hyperv_stimer0(regs);
+		break;
+#endif
+#ifdef CONFIG_ACRN_GUEST
+	case HYPERVISOR_CALLBACK_VECTOR:
+		__sysvec_acrn_hv_callback(regs);
+		break;
+#endif
+#ifdef CONFIG_XEN_PVHVM
+	case HYPERVISOR_CALLBACK_VECTOR:
+		__sysvec_xen_hvm_callback(regs);
+		break;
+#endif
+	case LOCAL_TIMER_VECTOR:
+		__sysvec_apic_timer_interrupt(regs);
+		break;
+	default:
+		printk_once(KERN_ERR "irq_pipeline: unexpected event"
+			" on vector #%.2x (irq=%u)", vector, irq);
+	}
+}
+
+static irqentry_state_t pipeline_enter_rcu(void)
+{
+	irqentry_state_t state = {
+		.exit_rcu = false,
+		.stage_info = IRQENTRY_INBAND_UNSTALLED,
+	};
+
+	if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
+		rcu_irq_enter();
+		state.exit_rcu = true;
+	} else {
+		rcu_irq_enter_check_tick();
+	}
+
+	return state;
+}
+
+static void pipeline_exit_rcu(irqentry_state_t state)
+{
+	if (state.exit_rcu)
+		rcu_irq_exit();
+}
+
+void arch_do_IRQ_pipelined(struct irq_desc *desc)
+{
+	struct pt_regs *regs = raw_cpu_ptr(&irq_pipeline.tick_regs);
+	struct pt_regs *old_regs = set_irq_regs(regs);
+	irqentry_state_t state;
+
+	/* Emulate a kernel entry. */
+	state = pipeline_enter_rcu();
+	irq_enter_rcu();
+
+	if (desc->irq_data.domain == sipic_domain)
+		run_irq_on_irqstack_cond(do_sysvec_inband, desc, regs);
+	else
+		run_irq_on_irqstack_cond(desc->handle_irq, desc, regs);
+
+	irq_exit_rcu();
+	pipeline_exit_rcu(state);
+
+	set_irq_regs(old_regs);
+}
+
+void arch_handle_irq(struct pt_regs *regs, u8 vector, bool irq_movable)
+{
+	struct irq_desc *desc;
+	unsigned int irq;
+
+	if (vector >= FIRST_SYSTEM_VECTOR) {
+		irq = apicm_vector_irq(vector);
+	} else {
+		desc = __this_cpu_read(vector_irq[vector]);
+		if (unlikely(IS_ERR_OR_NULL(desc))) {
+			__ack_APIC_irq();
+
+			if (desc == VECTOR_UNUSED) {
+				pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n",
+						__func__, smp_processor_id(),
+						vector);
+			} else {
+				__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
+			}
+			return;
+		}
+		if (irqd_is_setaffinity_pending(&desc->irq_data)) {
+			raw_spin_lock(&desc->lock);
+			if (irq_movable)
+				irqd_clr_move_blocked(&desc->irq_data);
+			else
+				irqd_set_move_blocked(&desc->irq_data);
+			raw_spin_unlock(&desc->lock);
+		}
+		irq = irq_desc_get_irq(desc);
+	}
+
+	generic_pipeline_irq(irq, regs);
+}
+
+noinstr void arch_pipeline_entry(struct pt_regs *regs, u8 vector)
+{
+	struct irq_stage_data *prevd;
+	irqentry_state_t state;
+
+	/*
+	 * The tricky one: we distinguish the following cases:
+	 *
+	 * [1] entry from oob context, either kernel or user code was
+	 * preempted by the IRQ, the in-band (virtual) interrupt state
+	 * is 'undefined' (could be either stalled/unstalled, it is
+	 * not relevant).
+	 *
+	 * [2] entry from in-band context while the stage is stalled,
+	 * which means that some kernel code was preempted by the IRQ
+	 * since in-band user code cannot run with interrupts
+	 * (virtually) disabled.
+	 *
+	 * [3] entry from in-band context while the stage is
+	 * unstalled: the common case for IRQ entry. Kernel or user
+	 * code may have been preempted, we handle the event
+	 * identically.
+	 *
+	 * [1] and [2] are processed almost the same way, except for
+	 * one key aspect: the potential stage demotion of the
+	 * preempted task which originally entered [1] on the oob
+	 * stage, then left it for the in-band stage as a result of
+	 * handling the IRQ (such demotion normally happens during
+	 * handle_irq_pipelined_finish() if required). In this
+	 * particular case, we want to run the common IRQ epilogue
+	 * code before returning to user mode, so that all pending
+	 * in-band work (_TIF_WORK_*) is carried out for the task
+	 * which is about to exit kernel mode.
+	 *
+	 * If the task runs in-band at the exit point and a user mode
+	 * context was preempted, then case [2] is excluded by
+	 * definition so we know for sure that we just observed a
+	 * stage demotion, therefore we have to run the work loop by
+	 * calling irqentry_exit_to_user_mode().
+	 */
+	if (unlikely(running_oob() || irqs_disabled())) {
+		instrumentation_begin();
+		prevd = handle_irq_pipelined_prepare(regs);
+		arch_handle_irq(regs, vector, false);
+		kvm_set_cpu_l1tf_flush_l1d();
+		handle_irq_pipelined_finish(prevd, regs);
+		if (running_inband() && user_mode(regs)) {
+			stall_inband_nocheck();
+			irqentry_exit_to_user_mode(regs);
+		}
+		instrumentation_end();
+		return;
+	}
+
+	/* In-band on entry, accepting interrupts. */
+	state = irqentry_enter(regs);
+	instrumentation_begin();
+	/* Prep for handling, switching oob. */
+	prevd = handle_irq_pipelined_prepare(regs);
+	arch_handle_irq(regs, vector, true);
+	kvm_set_cpu_l1tf_flush_l1d();
+	/* irqentry_enter() stalled the in-band stage. */
+	trace_hardirqs_on();
+	unstall_inband_nocheck();
+	handle_irq_pipelined_finish(prevd, regs);
+	stall_inband_nocheck();
+	trace_hardirqs_off();
+	instrumentation_end();
+	irqentry_exit(regs, state);
+}
+
+static int sipic_irq_map(struct irq_domain *d, unsigned int irq,
+			irq_hw_number_t hwirq)
+{
+	irq_set_percpu_devid(irq);
+	irq_set_chip_and_handler(irq, &sipic_chip, handle_apic_irq);
+
+	return 0;
+}
+
+static struct irq_domain_ops sipic_domain_ops = {
+	.map	= sipic_irq_map,
+};
+
+static void create_x86_apic_domain(void)
+{
+	sipic_domain = irq_domain_add_simple(NULL, NR_APIC_VECTORS,
+					     FIRST_SYSTEM_IRQ,
+					     &sipic_domain_ops, NULL);
+}
+
+#ifdef CONFIG_SMP
+
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(RESCHEDULE_OOB_VECTOR,
+				 sysvec_reschedule_oob_ipi)
+{ /* In-band handler is unused. */ }
+
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(TIMER_OOB_VECTOR,
+				 sysvec_timer_oob_ipi)
+{ /* In-band handler is unused. */ }
+
+void handle_irq_move_cleanup(struct irq_desc *desc)
+{
+	if (on_pipeline_entry()) {
+		/* 1. on receipt from hardware. */
+		__ack_APIC_irq();
+		handle_oob_irq(desc);
+	} else {
+		/* 2. in-band delivery. */
+		__sysvec_irq_move_cleanup(NULL);
+	}
+}
+
+static void smp_setup(void)
+{
+	int irq;
+
+	/*
+	 * The IRQ cleanup event must be pipelined to the inband
+	 * stage, so we need a valid IRQ descriptor for it. Since we
+	 * still are in the early boot stage on CPU0, we ask for a 1:1
+	 * mapping between the vector number and IRQ number, to make
+	 * things easier for us later on.
+	 */
+	irq = irq_alloc_desc_at(IRQ_MOVE_CLEANUP_VECTOR, 0);
+	WARN_ON(IRQ_MOVE_CLEANUP_VECTOR != irq);
+	/*
+	 * Set up the vector_irq[] mapping array for the boot CPU,
+	 * other CPUs will copy this entry when their APIC is going
+	 * online (see lapic_online()).
+	 */
+	per_cpu(vector_irq, 0)[irq] = irq_to_desc(irq);
+
+	irq_set_chip_and_handler(irq, &dummy_irq_chip,
+				handle_irq_move_cleanup);
+}
+
+#else
+
+static void smp_setup(void) { }
+
+#endif
+
+void __init arch_irq_pipeline_init(void)
+{
+	/*
+	 * Create an IRQ domain for mapping APIC system interrupts
+	 * (in-band and out-of-band), with fixed sirq numbers starting
+	 * from FIRST_SYSTEM_IRQ. Upon receipt of a system interrupt,
+	 * the corresponding sirq is injected into the pipeline.
+	 */
+	create_x86_apic_domain();
+
+	smp_setup();
+}
diff --git a/kernel/arch/x86/kernel/irq_work.c b/kernel/arch/x86/kernel/irq_work.c
index 890d4778cd..f2c8d141c6 100644
--- a/kernel/arch/x86/kernel/irq_work.c
+++ b/kernel/arch/x86/kernel/irq_work.c
@@ -14,7 +14,8 @@
 #include <linux/interrupt.h>
 
 #ifdef CONFIG_X86_LOCAL_APIC
-DEFINE_IDTENTRY_SYSVEC(sysvec_irq_work)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(IRQ_WORK_VECTOR,
+				 sysvec_irq_work)
 {
 	ack_APIC_irq();
 	trace_irq_work_entry(IRQ_WORK_VECTOR);
diff --git a/kernel/arch/x86/kernel/kvm.c b/kernel/arch/x86/kernel/kvm.c
index fe9babe948..6988375da1 100644
--- a/kernel/arch/x86/kernel/kvm.c
+++ b/kernel/arch/x86/kernel/kvm.c
@@ -255,12 +255,15 @@ noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
 {
 	u32 flags = kvm_read_and_reset_apf_flags();
 	irqentry_state_t state;
+	unsigned long irqflags;
 
 	if (!flags)
 		return false;
 
 	state = irqentry_enter(regs);
+	oob_trap_notify(X86_TRAP_PF, regs);
 	instrumentation_begin();
+	irqflags = hard_cond_local_irq_save();
 
 	/*
 	 * If the host managed to inject an async #PF into an interrupt
@@ -279,7 +282,9 @@ noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
 		WARN_ONCE(1, "Unexpected async PF flags: %x\n", flags);
 	}
 
+	hard_cond_local_irq_restore(irqflags);
 	instrumentation_end();
+	oob_trap_unwind(X86_TRAP_PF, regs);
 	irqentry_exit(regs, state);
 	return true;
 }
@@ -478,6 +483,9 @@ static DEFINE_PER_CPU(cpumask_var_t, __pv_cpu_mask);
 
 static void kvm_guest_cpu_offline(bool shutdown)
 {
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
 	kvm_disable_steal_time();
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
@@ -485,15 +493,16 @@ static void kvm_guest_cpu_offline(bool shutdown)
 	if (!shutdown)
 		apf_task_wake_all();
 	kvmclock_disable();
+	hard_local_irq_restore(flags);
 }
 
 static int kvm_cpu_online(unsigned int cpu)
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	local_irq_save_full(flags);
 	kvm_guest_cpu_init();
-	local_irq_restore(flags);
+	local_irq_restore_full(flags);
 	return 0;
 }
 
@@ -907,7 +916,7 @@ static void kvm_wait(u8 *ptr, u8 val)
 	if (in_nmi())
 		return;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	if (READ_ONCE(*ptr) != val)
 		goto out;
@@ -923,7 +932,7 @@ static void kvm_wait(u8 *ptr, u8 val)
 		safe_halt();
 
 out:
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 #ifdef CONFIG_X86_32
diff --git a/kernel/arch/x86/kernel/nmi.c b/kernel/arch/x86/kernel/nmi.c
index 2ef961cf4c..bf0766f6fd 100644
--- a/kernel/arch/x86/kernel/nmi.c
+++ b/kernel/arch/x86/kernel/nmi.c
@@ -473,6 +473,10 @@ static DEFINE_PER_CPU(enum nmi_states, nmi_state);
 static DEFINE_PER_CPU(unsigned long, nmi_cr2);
 static DEFINE_PER_CPU(unsigned long, nmi_dr7);
 
+/*
+ * IRQ pipeline: fixing up the virtual IRQ state makes no sense on
+ * NMI.
+ */
 DEFINE_IDTENTRY_RAW(exc_nmi)
 {
 	irqentry_state_t irq_state;
diff --git a/kernel/arch/x86/kernel/process.c b/kernel/arch/x86/kernel/process.c
index 5e17c3939d..a78266bffb 100644
--- a/kernel/arch/x86/kernel/process.c
+++ b/kernel/arch/x86/kernel/process.c
@@ -598,9 +598,9 @@ void speculation_ctrl_update(unsigned long tif)
 	unsigned long flags;
 
 	/* Forced update. Make sure all relevant TIF flags are different */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	__speculation_ctrl_update(~tif, tif);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /* Called from seccomp/prctl update */
@@ -700,6 +700,9 @@ void arch_cpu_idle(void)
 
 /*
  * We use this if we don't have any better idle routine..
+ *
+ * IRQ pipeline: safe_halt() returns with hard irqs on, caller does
+ * not need to force enable.
  */
 void __cpuidle default_idle(void)
 {
@@ -722,7 +725,7 @@ bool xen_set_default_idle(void)
 
 void stop_this_cpu(void *dummy)
 {
-	local_irq_disable();
+	hard_local_irq_disable();
 	/*
 	 * Remove this CPU:
 	 */
@@ -822,11 +825,14 @@ static __cpuidle void mwait_idle(void)
 		}
 
 		__monitor((void *)&current_thread_info()->flags, 0, 0);
-		if (!need_resched())
+		if (!need_resched()) {
 			__sti_mwait(0, 0);
-		else
+		} else {
+			hard_cond_local_irq_enable();
 			raw_local_irq_enable();
+		}
 	} else {
+		hard_cond_local_irq_enable();
 		raw_local_irq_enable();
 	}
 	__current_clr_polling();
diff --git a/kernel/arch/x86/kernel/process_64.c b/kernel/arch/x86/kernel/process_64.c
index ad3f82a18d..65d61712f1 100644
--- a/kernel/arch/x86/kernel/process_64.c
+++ b/kernel/arch/x86/kernel/process_64.c
@@ -272,9 +272,9 @@ void current_save_fsgs(void)
 	unsigned long flags;
 
 	/* Interrupts need to be off for FSGSBASE */
-	local_irq_save(flags);
+	local_irq_save_full(flags);
 	save_fsgs(current);
-	local_irq_restore(flags);
+	local_irq_restore_full(flags);
 }
 #if IS_ENABLED(CONFIG_KVM)
 EXPORT_SYMBOL_GPL(current_save_fsgs);
@@ -410,9 +410,9 @@ unsigned long x86_gsbase_read_cpu_inactive(void)
 	if (boot_cpu_has(X86_FEATURE_FSGSBASE)) {
 		unsigned long flags;
 
-		local_irq_save(flags);
+		local_irq_save_full(flags);
 		gsbase = __rdgsbase_inactive();
-		local_irq_restore(flags);
+		local_irq_restore_full(flags);
 	} else {
 		rdmsrl(MSR_KERNEL_GS_BASE, gsbase);
 	}
@@ -425,9 +425,9 @@ void x86_gsbase_write_cpu_inactive(unsigned long gsbase)
 	if (boot_cpu_has(X86_FEATURE_FSGSBASE)) {
 		unsigned long flags;
 
-		local_irq_save(flags);
+		local_irq_save_full(flags);
 		__wrgsbase_inactive(gsbase);
-		local_irq_restore(flags);
+		local_irq_restore_full(flags);
 	} else {
 		wrmsrl(MSR_KERNEL_GS_BASE, gsbase);
 	}
@@ -537,8 +537,17 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	struct thread_struct *next = &next_p->thread;
 	int cpu = smp_processor_id();
 
+	/*
+	 * Dovetail: Switching context on the out-of-band stage is
+	 * legit, and we may have preempted an in-band (soft)irq
+	 * handler earlier. Since oob handlers never switch stack,
+	 * make sure to restrict the following test to in-band
+	 * callers.
+	 */
 	WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
-		     this_cpu_read(irq_count) != -1);
+		     running_inband() && this_cpu_read(irq_count) != -1);
+
+	WARN_ON_ONCE(dovetail_debug() && !hard_irqs_disabled());
 
 	if (!test_thread_flag(TIF_NEED_FPU_LOAD))
 		switch_fpu_prepare(prev_p, cpu);
@@ -719,6 +728,7 @@ static long prctl_map_vdso(const struct vdso_image *image, unsigned long addr)
 
 long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
 {
+	unsigned long flags;
 	int ret = 0;
 
 	switch (option) {
@@ -726,7 +736,7 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
 		if (unlikely(arg2 >= TASK_SIZE_MAX))
 			return -EPERM;
 
-		preempt_disable();
+		flags = hard_preempt_disable();
 		/*
 		 * ARCH_SET_GS has always overwritten the index
 		 * and the base. Zero is the most sensible value
@@ -747,7 +757,7 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
 			task->thread.gsindex = 0;
 			x86_gsbase_write_task(task, arg2);
 		}
-		preempt_enable();
+		hard_preempt_enable(flags);
 		break;
 	}
 	case ARCH_SET_FS: {
@@ -758,7 +768,7 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
 		if (unlikely(arg2 >= TASK_SIZE_MAX))
 			return -EPERM;
 
-		preempt_disable();
+		flags = hard_preempt_disable();
 		/*
 		 * Set the selector to 0 for the same reason
 		 * as %gs above.
@@ -776,7 +786,7 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
 			task->thread.fsindex = 0;
 			x86_fsbase_write_task(task, arg2);
 		}
-		preempt_enable();
+		hard_preempt_enable(flags);
 		break;
 	}
 	case ARCH_GET_FS: {
diff --git a/kernel/arch/x86/kernel/smp.c b/kernel/arch/x86/kernel/smp.c
index eff4ce3b10..c4684dbbae 100644
--- a/kernel/arch/x86/kernel/smp.c
+++ b/kernel/arch/x86/kernel/smp.c
@@ -131,7 +131,7 @@ static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs)
 /*
  * this function calls the 'stop' function on all other CPUs in the system.
  */
-DEFINE_IDTENTRY_SYSVEC(sysvec_reboot)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(REBOOT_VECTOR, sysvec_reboot)
 {
 	ack_APIC_irq();
 	cpu_emergency_vmxoff();
@@ -212,17 +212,18 @@ static void native_stop_other_cpus(int wait)
 			udelay(1);
 	}
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	disable_local_APIC();
 	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /*
  * Reschedule call back. KVM uses this interrupt to force a cpu out of
  * guest mode.
  */
-DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi)
+DEFINE_IDTENTRY_SYSVEC_SIMPLE_PIPELINED(RESCHEDULE_VECTOR,
+					sysvec_reschedule_ipi)
 {
 	ack_APIC_irq();
 	trace_reschedule_entry(RESCHEDULE_VECTOR);
@@ -231,7 +232,8 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi)
 	trace_reschedule_exit(RESCHEDULE_VECTOR);
 }
 
-DEFINE_IDTENTRY_SYSVEC(sysvec_call_function)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(CALL_FUNCTION_VECTOR,
+				 sysvec_call_function)
 {
 	ack_APIC_irq();
 	trace_call_function_entry(CALL_FUNCTION_VECTOR);
@@ -240,7 +242,8 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function)
 	trace_call_function_exit(CALL_FUNCTION_VECTOR);
 }
 
-DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(CALL_FUNCTION_SINGLE_VECTOR,
+				 sysvec_call_function_single)
 {
 	ack_APIC_irq();
 	trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR);
diff --git a/kernel/arch/x86/kernel/smpboot.c b/kernel/arch/x86/kernel/smpboot.c
index e8e5515fb7..e9a7b3c0a5 100644
--- a/kernel/arch/x86/kernel/smpboot.c
+++ b/kernel/arch/x86/kernel/smpboot.c
@@ -258,7 +258,7 @@ static void notrace start_secondary(void *unused)
 	x86_platform.nmi_init();
 
 	/* enable local interrupts */
-	local_irq_enable();
+	local_irq_enable_full();
 
 	x86_cpuinit.setup_percpu_clockev();
 
@@ -1133,7 +1133,6 @@ int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
 {
 	int apicid = apic->cpu_present_to_apicid(cpu);
 	int cpu0_nmi_registered = 0;
-	unsigned long flags;
 	int err, ret = 0;
 
 	lockdep_assert_irqs_enabled();
@@ -1184,9 +1183,9 @@ int native_cpu_up(unsigned int cpu, struct task_struct *tidle)
 	 * Check TSC synchronization with the AP (keep irqs disabled
 	 * while doing so):
 	 */
-	local_irq_save(flags);
+	local_irq_disable_full();
 	check_tsc_sync_source(cpu);
-	local_irq_restore(flags);
+	local_irq_enable_full();
 
 	while (!cpu_online(cpu)) {
 		cpu_relax();
@@ -1654,7 +1653,7 @@ void play_dead_common(void)
 	/*
 	 * With physical CPU hotplug, we should halt the cpu
 	 */
-	local_irq_disable();
+	local_irq_disable_full();
 }
 
 /**
diff --git a/kernel/arch/x86/kernel/time.c b/kernel/arch/x86/kernel/time.c
index e42faa792c..874fd2ef13 100644
--- a/kernel/arch/x86/kernel/time.c
+++ b/kernel/arch/x86/kernel/time.c
@@ -54,7 +54,7 @@ EXPORT_SYMBOL(profile_pc);
  */
 static irqreturn_t timer_interrupt(int irq, void *dev_id)
 {
-	global_clock_event->event_handler(global_clock_event);
+	clockevents_handle_event(global_clock_event);
 	return IRQ_HANDLED;
 }
 
diff --git a/kernel/arch/x86/kernel/traps.c b/kernel/arch/x86/kernel/traps.c
index 2a39a2df6f..8ac977283c 100644
--- a/kernel/arch/x86/kernel/traps.c
+++ b/kernel/arch/x86/kernel/traps.c
@@ -74,14 +74,22 @@ DECLARE_BITMAP(system_vectors, NR_VECTORS);
 
 static inline void cond_local_irq_enable(struct pt_regs *regs)
 {
-	if (regs->flags & X86_EFLAGS_IF)
-		local_irq_enable();
+	if (regs->flags & X86_EFLAGS_IF) {
+		if (running_inband())
+			local_irq_enable_full();
+		else
+			hard_local_irq_enable();
+	}
 }
 
 static inline void cond_local_irq_disable(struct pt_regs *regs)
 {
-	if (regs->flags & X86_EFLAGS_IF)
-		local_irq_disable();
+	if (regs->flags & X86_EFLAGS_IF) {
+		if (running_inband())
+			local_irq_disable_full();
+		else
+			hard_local_irq_disable();
+	}
 }
 
 __always_inline int is_valid_bugaddr(unsigned long addr)
@@ -148,6 +156,39 @@ static void show_signal(struct task_struct *tsk, int signr,
 	}
 }
 
+static __always_inline
+bool mark_trap_entry(int trapnr, struct pt_regs *regs)
+{
+	oob_trap_notify(trapnr, regs);
+
+	if (likely(running_inband())) {
+		hard_cond_local_irq_enable();
+		return true;
+	}
+
+	return false;
+}
+
+static __always_inline
+void mark_trap_exit(int trapnr, struct pt_regs *regs)
+{
+	oob_trap_unwind(trapnr, regs);
+	hard_cond_local_irq_disable();
+}
+
+static __always_inline
+bool mark_trap_entry_raw(int trapnr, struct pt_regs *regs)
+{
+	oob_trap_notify(trapnr, regs);
+	return running_inband();
+}
+
+static __always_inline
+void mark_trap_exit_raw(int trapnr, struct pt_regs *regs)
+{
+	oob_trap_unwind(trapnr, regs);
+}
+
 static void
 do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
 	long error_code, int sicode, void __user *addr)
@@ -171,12 +212,17 @@ static void do_error_trap(struct pt_regs *regs, long error_code, char *str,
 {
 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
 
+	if (!mark_trap_entry(trapnr, regs))
+		return;
+
 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
 			NOTIFY_STOP) {
 		cond_local_irq_enable(regs);
 		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);
 		cond_local_irq_disable(regs);
 	}
+
+	mark_trap_exit(trapnr, regs);
 }
 
 /*
@@ -230,14 +276,22 @@ static noinstr bool handle_bug(struct pt_regs *regs)
 	 * Since we're emulating a CALL with exceptions, restore the interrupt
 	 * state to what it was at the exception site.
 	 */
-	if (regs->flags & X86_EFLAGS_IF)
-		raw_local_irq_enable();
+	if (regs->flags & X86_EFLAGS_IF) {
+		if (running_oob())
+			hard_local_irq_enable();
+		else
+			local_irq_enable_full();
+	}
 	if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {
 		regs->ip += LEN_UD2;
 		handled = true;
 	}
-	if (regs->flags & X86_EFLAGS_IF)
-		raw_local_irq_disable();
+	if (regs->flags & X86_EFLAGS_IF) {
+		if (running_oob())
+			hard_local_irq_disable();
+		else
+			local_irq_disable_full();
+	}
 	instrumentation_end();
 
 	return handled;
@@ -251,15 +305,26 @@ DEFINE_IDTENTRY_RAW(exc_invalid_op)
 	 * We use UD2 as a short encoding for 'CALL __WARN', as such
 	 * handle it before exception entry to avoid recursive WARN
 	 * in case exception entry is the one triggering WARNs.
+	 *
+	 * dovetail: handle_bug() may run oob, so we do not downgrade
+	 * in-band upon a failed __WARN assertion since it might have
+	 * tripped in a section of code which would not be happy to
+	 * switch stage. However, anything else should be notified to
+	 * the core, because the kernel execution might be about to
+	 * stop, so we'd need to switch in-band to get any output
+	 * before this happens.
 	 */
 	if (!user_mode(regs) && handle_bug(regs))
 		return;
 
-	state = irqentry_enter(regs);
-	instrumentation_begin();
-	handle_invalid_op(regs);
-	instrumentation_end();
-	irqentry_exit(regs, state);
+	if (mark_trap_entry_raw(X86_TRAP_UD, regs)) {
+		state = irqentry_enter(regs);
+		instrumentation_begin();
+		handle_invalid_op(regs);
+		instrumentation_end();
+		irqentry_exit(regs, state);
+		mark_trap_exit_raw(X86_TRAP_UD, regs);
+	}
 }
 
 DEFINE_IDTENTRY(exc_coproc_segment_overrun)
@@ -290,9 +355,12 @@ DEFINE_IDTENTRY_ERRORCODE(exc_alignment_check)
 {
 	char *str = "alignment check";
 
-	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)
+	if (!mark_trap_entry(X86_TRAP_AC, regs))
 		return;
 
+	if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP)
+		goto mark_exit;
+
 	if (!user_mode(regs))
 		die("Split lock detected\n", regs, error_code);
 
@@ -306,6 +374,9 @@ DEFINE_IDTENTRY_ERRORCODE(exc_alignment_check)
 
 out:
 	local_irq_disable();
+
+mark_exit:
+	mark_trap_exit(X86_TRAP_AC, regs);
 }
 
 #ifdef CONFIG_VMAP_STACK
@@ -341,6 +412,9 @@ __visible void __noreturn handle_stack_overflow(const char *message,
  *
  * The 32bit #DF shim provides CR2 already as an argument. On 64bit it needs
  * to be read before doing anything else.
+ *
+ * Dovetail: do not even ask the companion core to try restoring the
+ * in-band stage on double-fault, this would be a lost cause.
  */
 DEFINE_IDTENTRY_DF(exc_double_fault)
 {
@@ -465,9 +539,12 @@ DEFINE_IDTENTRY_DF(exc_double_fault)
 
 DEFINE_IDTENTRY(exc_bounds)
 {
+	if (!mark_trap_entry(X86_TRAP_BR, regs))
+		return;
+
 	if (notify_die(DIE_TRAP, "bounds", regs, 0,
 			X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP)
-		return;
+		goto out;
 	cond_local_irq_enable(regs);
 
 	if (!user_mode(regs))
@@ -476,6 +553,8 @@ DEFINE_IDTENTRY(exc_bounds)
 	do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL);
 
 	cond_local_irq_disable(regs);
+out:
+	mark_trap_exit(X86_TRAP_BR, regs);
 }
 
 enum kernel_gp_hint {
@@ -570,9 +649,9 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
 	}
 
 	if (v8086_mode(regs)) {
-		local_irq_enable();
+		local_irq_enable_full();
 		handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
-		local_irq_disable();
+		local_irq_disable_full();
 		return;
 	}
 
@@ -585,9 +664,12 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
 		tsk->thread.error_code = error_code;
 		tsk->thread.trap_nr = X86_TRAP_GP;
 
+		if (!mark_trap_entry(X86_TRAP_GP, regs))
+			goto exit;
+
 		show_signal(tsk, SIGSEGV, "", desc, regs, error_code);
 		force_sig(SIGSEGV);
-		goto exit;
+		goto mark_exit;
 	}
 
 	if (fixup_exception(regs, X86_TRAP_GP, error_code, 0))
@@ -605,9 +687,12 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
 	    kprobe_fault_handler(regs, X86_TRAP_GP))
 		goto exit;
 
+	if (!mark_trap_entry(X86_TRAP_GP, regs))
+		goto exit;
+
 	ret = notify_die(DIE_GPF, desc, regs, error_code, X86_TRAP_GP, SIGSEGV);
 	if (ret == NOTIFY_STOP)
-		goto exit;
+		goto mark_exit;
 
 	if (error_code)
 		snprintf(desc, sizeof(desc), "segment-related " GPFSTR);
@@ -629,6 +714,8 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection)
 
 	die_addr(desc, regs, error_code, gp_addr);
 
+mark_exit:
+	mark_trap_exit(X86_TRAP_GP, regs);
 exit:
 	cond_local_irq_disable(regs);
 }
@@ -673,6 +760,9 @@ DEFINE_IDTENTRY_RAW(exc_int3)
 	if (poke_int3_handler(regs))
 		return;
 
+	if (!mark_trap_entry_raw(X86_TRAP_BP, regs))
+		return;
+
 	/*
 	 * irqentry_enter_from_user_mode() uses static_branch_{,un}likely()
 	 * and therefore can trigger INT3, hence poke_int3_handler() must
@@ -695,6 +785,8 @@ DEFINE_IDTENTRY_RAW(exc_int3)
 		instrumentation_end();
 		irqentry_nmi_exit(regs, irq_state);
 	}
+
+	mark_trap_exit_raw(X86_TRAP_BP, regs);
 }
 
 #ifdef CONFIG_X86_64
@@ -999,7 +1091,7 @@ static __always_inline void exc_debug_user(struct pt_regs *regs,
 		goto out;
 
 	/* It's safe to allow irq's after DR6 has been saved */
-	local_irq_enable();
+	local_irq_enable_full();
 
 	if (v8086_mode(regs)) {
 		handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);
@@ -1012,7 +1104,7 @@ static __always_inline void exc_debug_user(struct pt_regs *regs,
 		send_sigtrap(regs, 0, get_si_code(dr6));
 
 out_irq:
-	local_irq_disable();
+	local_irq_disable_full();
 out:
 	instrumentation_end();
 	irqentry_exit_to_user_mode(regs);
@@ -1022,13 +1114,19 @@ static __always_inline void exc_debug_user(struct pt_regs *regs,
 /* IST stack entry */
 DEFINE_IDTENTRY_DEBUG(exc_debug)
 {
-	exc_debug_kernel(regs, debug_read_clear_dr6());
+	if (mark_trap_entry_raw(X86_TRAP_DB, regs)) {
+		exc_debug_kernel(regs, debug_read_clear_dr6());
+		mark_trap_exit_raw(X86_TRAP_DB, regs);
+	}
 }
 
 /* User entry, runs on regular task stack */
 DEFINE_IDTENTRY_DEBUG_USER(exc_debug)
 {
-	exc_debug_user(regs, debug_read_clear_dr6());
+	if (mark_trap_entry_raw(X86_TRAP_DB, regs)) {
+		exc_debug_user(regs, debug_read_clear_dr6());
+		mark_trap_exit_raw(X86_TRAP_DB, regs);
+	}
 }
 #else
 /* 32 bit does not have separate entry points. */
@@ -1062,13 +1160,16 @@ static void math_error(struct pt_regs *regs, int trapnr)
 		if (fixup_exception(regs, trapnr, 0, 0))
 			goto exit;
 
+		if (!mark_trap_entry(trapnr, regs))
+			goto exit;
+
 		task->thread.error_code = 0;
 		task->thread.trap_nr = trapnr;
 
 		if (notify_die(DIE_TRAP, str, regs, 0, trapnr,
 			       SIGFPE) != NOTIFY_STOP)
 			die(str, regs, 0);
-		goto exit;
+		goto mark_exit;
 	}
 
 	/*
@@ -1084,8 +1185,13 @@ static void math_error(struct pt_regs *regs, int trapnr)
 	if (!si_code)
 		goto exit;
 
+	if (!mark_trap_entry(trapnr, regs))
+		goto exit;
+
 	force_sig_fault(SIGFPE, si_code,
 			(void __user *)uprobe_get_trap_addr(regs));
+mark_exit:
+	mark_trap_exit(trapnr, regs);
 exit:
 	cond_local_irq_disable(regs);
 }
@@ -1158,7 +1264,10 @@ DEFINE_IDTENTRY(exc_device_not_available)
 		 * to kill the task than getting stuck in a never-ending
 		 * loop of #NM faults.
 		 */
-		die("unexpected #NM exception", regs, 0);
+		if (mark_trap_entry(X86_TRAP_NM, regs)) {
+			die("unexpected #NM exception", regs, 0);
+			mark_trap_exit(X86_TRAP_NM, regs);
+		}
 	}
 }
 
diff --git a/kernel/arch/x86/kernel/tsc.c b/kernel/arch/x86/kernel/tsc.c
index f9f1b45e5d..2b59b3002f 100644
--- a/kernel/arch/x86/kernel/tsc.c
+++ b/kernel/arch/x86/kernel/tsc.c
@@ -131,8 +131,11 @@ static void __set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long ts
 {
 	unsigned long long ns_now;
 	struct cyc2ns_data data;
+	unsigned long flags;
 	struct cyc2ns *c2n;
 
+	flags = hard_cond_local_irq_save();
+
 	ns_now = cycles_2_ns(tsc_now);
 
 	/*
@@ -163,6 +166,8 @@ static void __set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long ts
 	c2n->data[0] = data;
 	raw_write_seqcount_latch(&c2n->seq);
 	c2n->data[1] = data;
+
+	hard_cond_local_irq_restore(flags);
 }
 
 static void set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now)
@@ -759,11 +764,11 @@ static unsigned long pit_hpet_ptimer_calibrate_cpu(void)
 		 * calibration, which will take at least 50ms, and
 		 * read the end value.
 		 */
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 		tsc1 = tsc_read_refs(&ref1, hpet);
 		tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin);
 		tsc2 = tsc_read_refs(&ref2, hpet);
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 
 		/* Pick the lowest PIT TSC calibration so far */
 		tsc_pit_min = min(tsc_pit_min, tsc_pit_khz);
@@ -872,9 +877,9 @@ unsigned long native_calibrate_cpu_early(void)
 	if (!fast_calibrate)
 		fast_calibrate = cpu_khz_from_msr();
 	if (!fast_calibrate) {
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 		fast_calibrate = quick_pit_calibrate();
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 	}
 	return fast_calibrate;
 }
@@ -942,7 +947,7 @@ void tsc_restore_sched_clock_state(void)
 	if (!sched_clock_stable())
 		return;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	/*
 	 * We're coming out of suspend, there's no concurrency yet; don't
@@ -960,7 +965,7 @@ void tsc_restore_sched_clock_state(void)
 		per_cpu(cyc2ns.data[1].cyc2ns_offset, cpu) = offset;
 	}
 
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 #ifdef CONFIG_CPU_FREQ
@@ -1411,6 +1416,8 @@ static int __init init_tsc_clocksource(void)
 	if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3))
 		clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
 
+	clocksource_tsc.vdso_type = CLOCKSOURCE_VDSO_ARCHITECTED;
+
 	/*
 	 * When TSC frequency is known (retrieved via MSR or CPUID), we skip
 	 * the refined calibration and directly register it as a clocksource.
diff --git a/kernel/arch/x86/kernel/tsc_sync.c b/kernel/arch/x86/kernel/tsc_sync.c
index 923660057f..883c0df13a 100644
--- a/kernel/arch/x86/kernel/tsc_sync.c
+++ b/kernel/arch/x86/kernel/tsc_sync.c
@@ -367,6 +367,8 @@ void check_tsc_sync_source(int cpu)
 		atomic_set(&test_runs, 1);
 	else
 		atomic_set(&test_runs, 3);
+
+	hard_cond_local_irq_disable();
 retry:
 	/*
 	 * Wait for the target to start or to skip the test:
@@ -448,6 +450,8 @@ void check_tsc_sync_target(void)
 	if (unsynchronized_tsc())
 		return;
 
+	hard_cond_local_irq_disable();
+
 	/*
 	 * Store, verify and sanitize the TSC adjust register. If
 	 * successful skip the test.
diff --git a/kernel/arch/x86/kvm/emulate.c b/kernel/arch/x86/kvm/emulate.c
index 63efccc8f4..4301a49fcc 100644
--- a/kernel/arch/x86/kvm/emulate.c
+++ b/kernel/arch/x86/kvm/emulate.c
@@ -1125,23 +1125,27 @@ static void fetch_register_operand(struct operand *op)
 	}
 }
 
-static void emulator_get_fpu(void)
+static unsigned long emulator_get_fpu(void)
 {
-	fpregs_lock();
+	unsigned long flags = fpregs_lock();
 
 	fpregs_assert_state_consistent();
 	if (test_thread_flag(TIF_NEED_FPU_LOAD))
 		switch_fpu_return();
+
+	return flags;
 }
 
-static void emulator_put_fpu(void)
+static void emulator_put_fpu(unsigned long flags)
 {
-	fpregs_unlock();
+	fpregs_unlock(flags);
 }
 
 static void read_sse_reg(sse128_t *data, int reg)
 {
-	emulator_get_fpu();
+	unsigned long flags;
+
+	flags = emulator_get_fpu();
 	switch (reg) {
 	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
 	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
@@ -1163,12 +1167,14 @@ static void read_sse_reg(sse128_t *data, int reg)
 #endif
 	default: BUG();
 	}
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 }
 
 static void write_sse_reg(sse128_t *data, int reg)
 {
-	emulator_get_fpu();
+	unsigned long flags;
+
+	flags = emulator_get_fpu();
 	switch (reg) {
 	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
 	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
@@ -1190,12 +1196,14 @@ static void write_sse_reg(sse128_t *data, int reg)
 #endif
 	default: BUG();
 	}
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 }
 
 static void read_mmx_reg(u64 *data, int reg)
 {
-	emulator_get_fpu();
+	unsigned long flags;
+
+	flags = emulator_get_fpu();
 	switch (reg) {
 	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
 	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
@@ -1207,12 +1215,14 @@ static void read_mmx_reg(u64 *data, int reg)
 	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
 	default: BUG();
 	}
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 }
 
 static void write_mmx_reg(u64 *data, int reg)
 {
-	emulator_get_fpu();
+	unsigned long flags;
+
+	flags = emulator_get_fpu();
 	switch (reg) {
 	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
 	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
@@ -1224,30 +1234,33 @@ static void write_mmx_reg(u64 *data, int reg)
 	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
 	default: BUG();
 	}
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 }
 
 static int em_fninit(struct x86_emulate_ctxt *ctxt)
 {
+	unsigned long flags;
+
 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
 		return emulate_nm(ctxt);
 
-	emulator_get_fpu();
+	flags = emulator_get_fpu();
 	asm volatile("fninit");
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 	return X86EMUL_CONTINUE;
 }
 
 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
 {
+	unsigned long flags;
 	u16 fcw;
 
 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
 		return emulate_nm(ctxt);
 
-	emulator_get_fpu();
+	flags = emulator_get_fpu();
 	asm volatile("fnstcw %0": "+m"(fcw));
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 
 	ctxt->dst.val = fcw;
 
@@ -1256,14 +1269,15 @@ static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
 
 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
 {
+	unsigned long flags;
 	u16 fsw;
 
 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
 		return emulate_nm(ctxt);
 
-	emulator_get_fpu();
+	flags = emulator_get_fpu();
 	asm volatile("fnstsw %0": "+m"(fsw));
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 
 	ctxt->dst.val = fsw;
 
@@ -4182,17 +4196,18 @@ static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
 {
 	struct fxregs_state fx_state;
+	unsigned long flags;
 	int rc;
 
 	rc = check_fxsr(ctxt);
 	if (rc != X86EMUL_CONTINUE)
 		return rc;
 
-	emulator_get_fpu();
+	flags = emulator_get_fpu();
 
 	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
 
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 
 	if (rc != X86EMUL_CONTINUE)
 		return rc;
@@ -4224,6 +4239,7 @@ static noinline int fxregs_fixup(struct fxregs_state *fx_state,
 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
 {
 	struct fxregs_state fx_state;
+	unsigned long flags;
 	int rc;
 	size_t size;
 
@@ -4236,7 +4252,7 @@ static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
 	if (rc != X86EMUL_CONTINUE)
 		return rc;
 
-	emulator_get_fpu();
+	flags = emulator_get_fpu();
 
 	if (size < __fxstate_size(16)) {
 		rc = fxregs_fixup(&fx_state, size);
@@ -4253,7 +4269,7 @@ static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
 		rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
 
 out:
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 
 	return rc;
 }
@@ -5498,11 +5514,12 @@ static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
 
 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
 {
+	unsigned long flags;
 	int rc;
 
-	emulator_get_fpu();
+	flags = emulator_get_fpu();
 	rc = asm_safe("fwait");
-	emulator_put_fpu();
+	emulator_put_fpu(flags);
 
 	if (unlikely(rc != X86EMUL_CONTINUE))
 		return emulate_exception(ctxt, MF_VECTOR, 0, false);
diff --git a/kernel/arch/x86/kvm/vmx/vmx.c b/kernel/arch/x86/kvm/vmx/vmx.c
index af6742d11c..4554a63036 100644
--- a/kernel/arch/x86/kvm/vmx/vmx.c
+++ b/kernel/arch/x86/kvm/vmx/vmx.c
@@ -752,14 +752,15 @@ struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr)
 static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx,
 				  struct vmx_uret_msr *msr, u64 data)
 {
+	unsigned long flags;
 	int ret = 0;
 
 	u64 old_msr_data = msr->data;
 	msr->data = data;
 	if (msr - vmx->guest_uret_msrs < vmx->nr_active_uret_msrs) {
-		preempt_disable();
+		flags = hard_preempt_disable();
 		ret = kvm_set_user_return_msr(msr->slot, msr->data, msr->mask);
-		preempt_enable();
+		hard_preempt_enable(flags);
 		if (ret)
 			msr->data = old_msr_data;
 	}
@@ -1383,19 +1384,23 @@ static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
 #ifdef CONFIG_X86_64
 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
 {
-	preempt_disable();
+	unsigned long flags;
+
+	flags = hard_preempt_disable();
 	if (vmx->guest_state_loaded)
 		rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
-	preempt_enable();
+	hard_preempt_enable(flags);
 	return vmx->msr_guest_kernel_gs_base;
 }
 
 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
 {
-	preempt_disable();
+	unsigned long flags;
+
+	flags = hard_preempt_disable();
 	if (vmx->guest_state_loaded)
 		wrmsrl(MSR_KERNEL_GS_BASE, data);
-	preempt_enable();
+	hard_preempt_enable(flags);
 	vmx->msr_guest_kernel_gs_base = data;
 }
 #endif
@@ -1795,6 +1800,7 @@ static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr)
  */
 static void setup_msrs(struct vcpu_vmx *vmx)
 {
+	hard_cond_local_irq_disable();
 	vmx->guest_uret_msrs_loaded = false;
 	vmx->nr_active_uret_msrs = 0;
 #ifdef CONFIG_X86_64
@@ -1815,6 +1821,7 @@ static void setup_msrs(struct vcpu_vmx *vmx)
 		vmx_setup_uret_msr(vmx, MSR_TSC_AUX);
 
 	vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL);
+	hard_cond_local_irq_enable();
 
 	if (cpu_has_vmx_msr_bitmap())
 		vmx_update_msr_bitmap(&vmx->vcpu);
@@ -2050,6 +2057,7 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	u32 msr_index = msr_info->index;
 	u64 data = msr_info->data;
 	u32 index;
+	unsigned long flags;
 
 	switch (msr_index) {
 	case MSR_EFER:
@@ -2289,11 +2297,22 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 
 	default:
 	find_uret_msr:
+		/*
+		 * Guest MSRs may be activated independently from
+		 * vcpu_run(): rely on the notifier for restoring them
+		 * upon preemption by the companion core, right before
+		 * the current CPU switches to out-of-band scheduling
+		 * (see dovetail_context_switch()).
+		 */
 		msr = vmx_find_uret_msr(vmx, msr_index);
-		if (msr)
+		if (msr) {
+			flags = hard_cond_local_irq_save();
+			inband_enter_guest(vcpu);
 			ret = vmx_set_guest_uret_msr(vmx, msr, data);
-		else
+			hard_cond_local_irq_restore(flags);
+		} else {
 			ret = kvm_set_msr_common(vcpu, msr_info);
+		}
 	}
 
 	/* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */
@@ -7056,7 +7075,9 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
 	vmx_vcpu_load(vcpu, cpu);
 	vcpu->cpu = cpu;
 	init_vmcs(vmx);
+	hard_cond_local_irq_disable();
 	vmx_vcpu_put(vcpu);
+	hard_cond_local_irq_enable();
 	put_cpu();
 	if (cpu_need_virtualize_apic_accesses(vcpu)) {
 		err = alloc_apic_access_page(vcpu->kvm);
diff --git a/kernel/arch/x86/kvm/x86.c b/kernel/arch/x86/kvm/x86.c
index 23d7c563e0..7928751435 100644
--- a/kernel/arch/x86/kvm/x86.c
+++ b/kernel/arch/x86/kvm/x86.c
@@ -178,6 +178,7 @@ struct kvm_user_return_msrs_global {
 struct kvm_user_return_msrs {
 	struct user_return_notifier urn;
 	bool registered;
+	bool dirty;
 	struct kvm_user_return_msr_values {
 		u64 host;
 		u64 curr;
@@ -295,12 +296,29 @@ static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
 		vcpu->arch.apf.gfns[i] = ~0;
 }
 
-static void kvm_on_user_return(struct user_return_notifier *urn)
+static void __kvm_on_user_return(struct kvm_user_return_msrs *msrs)
 {
+	struct kvm_user_return_msr_values *values;
 	unsigned slot;
+
+	if (!msrs->dirty)
+		return;
+
+	for (slot = 0; slot < user_return_msrs_global.nr; ++slot) {
+		values = &msrs->values[slot];
+		if (values->host != values->curr) {
+			wrmsrl(user_return_msrs_global.msrs[slot], values->host);
+			values->curr = values->host;
+		}
+	}
+
+	msrs->dirty = false;
+}
+
+static void kvm_on_user_return(struct user_return_notifier *urn)
+{
 	struct kvm_user_return_msrs *msrs
 		= container_of(urn, struct kvm_user_return_msrs, urn);
-	struct kvm_user_return_msr_values *values;
 	unsigned long flags;
 
 	/*
@@ -313,13 +331,10 @@ static void kvm_on_user_return(struct user_return_notifier *urn)
 		user_return_notifier_unregister(urn);
 	}
 	local_irq_restore(flags);
-	for (slot = 0; slot < user_return_msrs_global.nr; ++slot) {
-		values = &msrs->values[slot];
-		if (values->host != values->curr) {
-			wrmsrl(user_return_msrs_global.msrs[slot], values->host);
-			values->curr = values->host;
-		}
-	}
+	flags = hard_cond_local_irq_save();
+	__kvm_on_user_return(msrs);
+	hard_cond_local_irq_restore(flags);
+	inband_exit_guest();
 }
 
 int kvm_probe_user_return_msr(u32 msr)
@@ -374,6 +389,7 @@ int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
 	if (err)
 		return 1;
 
+	msrs->dirty = true;
 	msrs->values[slot].curr = value;
 	if (!msrs->registered) {
 		msrs->urn.on_user_return = kvm_on_user_return;
@@ -4072,11 +4088,23 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
+	struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);
+	unsigned long flags;
 	int idx;
 
 	if (vcpu->preempted)
 		vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu);
 
+	flags = hard_cond_local_irq_save();
+	/*
+	 * Skip steal time accounting from the out-of-band stage since
+	 * this is oob-unsafe. We leave it to the next call from the
+	 * inband stage.
+	 */
+	if (running_oob())
+		goto skip_steal_time_update;
+
+
 	/*
 	 * Disable page faults because we're in atomic context here.
 	 * kvm_write_guest_offset_cached() would call might_fault()
@@ -4094,6 +4122,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 	kvm_steal_time_set_preempted(vcpu);
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 	pagefault_enable();
+skip_steal_time_update:
 	kvm_x86_ops.vcpu_put(vcpu);
 	vcpu->arch.last_host_tsc = rdtsc();
 	/*
@@ -4102,8 +4131,41 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
 	 */
 	set_debugreg(0, 6);
+
+	inband_set_vcpu_release_state(vcpu, false);
+	if (!msrs->dirty)
+		inband_exit_guest();
+
+	hard_cond_local_irq_restore(flags);
 }
 
+#ifdef CONFIG_DOVETAIL
+/* hard irqs off. */
+void kvm_handle_oob_switch(struct kvm_oob_notifier *nfy)
+{
+	struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);
+	struct kvm_vcpu *vcpu;
+
+	vcpu = container_of(nfy, struct kvm_vcpu, oob_notifier);
+	/*
+	 * If user_return MSRs were still active when leaving
+	 * kvm_arch_vcpu_put(), inband_exit_guest() was not invoked,
+	 * so we might get called later on before kvm_on_user_return()
+	 * had a chance to run, if a switch to out-of-band scheduling
+	 * sneaks in in the meantime.  Prevent kvm_arch_vcpu_put()
+	 * from running twice in such a case by checking ->put_vcpu
+	 * from the notifier block.
+	 */
+	if (nfy->put_vcpu)
+		kvm_arch_vcpu_put(vcpu);
+
+	__kvm_on_user_return(msrs);
+	inband_exit_guest();
+}
+#else
+#define kvm_handle_oob_switch  NULL
+#endif
+
 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
 				    struct kvm_lapic_state *s)
 {
@@ -9142,6 +9204,10 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	}
 
 	preempt_disable();
+	local_irq_disable_full();
+
+	inband_enter_guest(vcpu);
+	inband_set_vcpu_release_state(vcpu, true);
 
 	kvm_x86_ops.prepare_guest_switch(vcpu);
 
@@ -9150,7 +9216,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	 * IPI are then delayed after guest entry, which ensures that they
 	 * result in virtual interrupt delivery.
 	 */
-	local_irq_disable();
 	vcpu->mode = IN_GUEST_MODE;
 
 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
@@ -9179,7 +9244,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	if (kvm_vcpu_exit_request(vcpu)) {
 		vcpu->mode = OUTSIDE_GUEST_MODE;
 		smp_wmb();
-		local_irq_enable();
+		local_irq_enable_full();
 		preempt_enable();
 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 		r = 1;
@@ -9251,9 +9316,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	 * stat.exits increment will do nicely.
 	 */
 	kvm_before_interrupt(vcpu);
-	local_irq_enable();
+	local_irq_enable_full();
 	++vcpu->stat.exits;
-	local_irq_disable();
+	local_irq_disable_full();
 	kvm_after_interrupt(vcpu);
 
 	/*
@@ -9273,7 +9338,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 		}
 	}
 
-	local_irq_enable();
+	local_irq_enable_full();
 	preempt_enable();
 
 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
@@ -9487,7 +9552,9 @@ static void kvm_save_current_fpu(struct fpu *fpu)
 /* Swap (qemu) user FPU context for the guest FPU context. */
 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 {
-	fpregs_lock();
+	unsigned long flags;
+
+	flags = fpregs_lock();
 
 	kvm_save_current_fpu(vcpu->arch.user_fpu);
 
@@ -9496,7 +9563,7 @@ static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 				~XFEATURE_MASK_PKRU);
 
 	fpregs_mark_activate();
-	fpregs_unlock();
+	fpregs_unlock(flags);
 
 	trace_kvm_fpu(1);
 }
@@ -9504,14 +9571,16 @@ static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 /* When vcpu_run ends, restore user space FPU context. */
 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
 {
-	fpregs_lock();
+	unsigned long flags;
+
+	flags = fpregs_lock();
 
 	kvm_save_current_fpu(vcpu->arch.guest_fpu);
 
 	copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
 
 	fpregs_mark_activate();
-	fpregs_unlock();
+	fpregs_unlock(flags);
 
 	++vcpu->stat.fpu_reload;
 	trace_kvm_fpu(0);
@@ -10189,6 +10258,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 	if (r)
 		goto free_guest_fpu;
 
+	inband_init_vcpu(vcpu, kvm_handle_oob_switch);
 	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
 	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
 	kvm_vcpu_mtrr_init(vcpu);
diff --git a/kernel/arch/x86/lib/usercopy.c b/kernel/arch/x86/lib/usercopy.c
index 3f435d7fca..1cdb806bd6 100644
--- a/kernel/arch/x86/lib/usercopy.c
+++ b/kernel/arch/x86/lib/usercopy.c
@@ -18,7 +18,7 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
 {
 	unsigned long ret;
 
-	if (__range_not_ok(from, n, TASK_SIZE))
+	if (running_oob() || __range_not_ok(from, n, TASK_SIZE))
 		return n;
 
 	if (!nmi_uaccess_okay())
diff --git a/kernel/arch/x86/mm/fault.c b/kernel/arch/x86/mm/fault.c
index e9afbf85e6..a4d3b1851d 100644
--- a/kernel/arch/x86/mm/fault.c
+++ b/kernel/arch/x86/mm/fault.c
@@ -19,6 +19,7 @@
 #include <linux/uaccess.h>		/* faulthandler_disabled()	*/
 #include <linux/efi.h>			/* efi_recover_from_page_fault()*/
 #include <linux/mm_types.h>
+#include <linux/irqstage.h>
 
 #include <asm/cpufeature.h>		/* boot_cpu_has, ...		*/
 #include <asm/traps.h>			/* dotraplinkage, ...		*/
@@ -656,7 +657,7 @@ no_context(struct pt_regs *regs, unsigned long error_code,
 		 * the below recursive fault logic only apply to a faults from
 		 * task context.
 		 */
-		if (in_interrupt())
+		if (running_oob() || in_interrupt())
 			return;
 
 		/*
@@ -666,10 +667,12 @@ no_context(struct pt_regs *regs, unsigned long error_code,
 		 * faulting through the emulate_vsyscall() logic.
 		 */
 		if (current->thread.sig_on_uaccess_err && signal) {
+			oob_trap_notify(X86_TRAP_PF, regs);
 			set_signal_archinfo(address, error_code);
 
 			/* XXX: hwpoison faults will set the wrong code. */
 			force_sig_fault(signal, si_code, (void __user *)address);
+			oob_trap_unwind(X86_TRAP_PF, regs);
 		}
 
 		/*
@@ -678,6 +681,12 @@ no_context(struct pt_regs *regs, unsigned long error_code,
 		return;
 	}
 
+	/*
+	 * Do not bother unwinding the notification context on
+	 * CPU/firmware/kernel bug.
+	 */
+	oob_trap_notify(X86_TRAP_PF, regs);
+
 #ifdef CONFIG_VMAP_STACK
 	/*
 	 * Stack overflow?  During boot, we can fault near the initial
@@ -796,6 +805,55 @@ static bool is_vsyscall_vaddr(unsigned long vaddr)
 	return unlikely((vaddr & PAGE_MASK) == VSYSCALL_ADDR);
 }
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+static inline void cond_reenable_irqs_user(void)
+{
+	hard_local_irq_enable();
+
+	if (running_inband())
+		local_irq_enable();
+}
+
+static inline void cond_reenable_irqs_kernel(irqentry_state_t state,
+					struct pt_regs *regs)
+{
+	if (regs->flags & X86_EFLAGS_IF) {
+		hard_local_irq_enable();
+		if (state.stage_info == IRQENTRY_INBAND_UNSTALLED)
+			local_irq_enable();
+	}
+}
+
+static inline void cond_disable_irqs(void)
+{
+	hard_local_irq_disable();
+
+	if (running_inband())
+		local_irq_disable();
+}
+
+#else  /* !CONFIG_IRQ_PIPELINE */
+
+static inline void cond_reenable_irqs_user(void)
+{
+	local_irq_enable();
+}
+
+static inline void cond_reenable_irqs_kernel(irqentry_state_t state,
+					struct pt_regs *regs)
+{
+	if (regs->flags & X86_EFLAGS_IF)
+		local_irq_enable();
+}
+
+static inline void cond_disable_irqs(void)
+{
+	local_irq_disable();
+}
+
+#endif  /* !CONFIG_IRQ_PIPELINE */
+
 static void
 __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 		       unsigned long address, u32 pkey, int si_code)
@@ -807,7 +865,7 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 		/*
 		 * It's possible to have interrupts off here:
 		 */
-		local_irq_enable();
+		cond_reenable_irqs_user();
 
 		/*
 		 * Valid to do another page fault here because this one came
@@ -819,6 +877,12 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 		if (is_errata100(regs, address))
 			return;
 
+		oob_trap_notify(X86_TRAP_PF, regs);
+		if (!running_inband()) {
+			local_irq_disable_full();
+			return;
+		}
+
 		/*
 		 * To avoid leaking information about the kernel page table
 		 * layout, pretend that user-mode accesses to kernel addresses
@@ -837,7 +901,9 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
 
 		force_sig_fault(SIGSEGV, si_code, (void __user *)address);
 
-		local_irq_disable();
+		local_irq_disable_full();
+
+		oob_trap_unwind(X86_TRAP_PF, regs);
 
 		return;
 	}
@@ -1225,7 +1291,8 @@ NOKPROBE_SYMBOL(do_kern_addr_fault);
 static inline
 void do_user_addr_fault(struct pt_regs *regs,
 			unsigned long hw_error_code,
-			unsigned long address)
+			unsigned long address,
+			irqentry_state_t state)
 {
 	struct vm_area_struct *vma = NULL;
 	struct task_struct *tsk;
@@ -1266,7 +1333,7 @@ void do_user_addr_fault(struct pt_regs *regs,
 	 * If we're in an interrupt, have no user context or are running
 	 * in a region with pagefaults disabled then we must not take the fault
 	 */
-	if (unlikely(faulthandler_disabled() || !mm)) {
+	if (unlikely(running_inband() && (faulthandler_disabled() || !mm))) {
 		bad_area_nosemaphore(regs, hw_error_code, address);
 		return;
 	}
@@ -1279,13 +1346,23 @@ void do_user_addr_fault(struct pt_regs *regs,
 	 * potential system fault or CPU buglet:
 	 */
 	if (user_mode(regs)) {
-		local_irq_enable();
+		cond_reenable_irqs_user();
 		flags |= FAULT_FLAG_USER;
 	} else {
-		if (regs->flags & X86_EFLAGS_IF)
-			local_irq_enable();
+		cond_reenable_irqs_kernel(state, regs);
 	}
 
+	/*
+	 * At this point, we would have to stop running
+	 * out-of-band. Tell the companion core about the page fault
+	 * event, so that it might switch current to in-band mode if
+	 * need be. If it does not, then we may assume that it would
+	 * also handle the fixups.
+	 */
+	oob_trap_notify(X86_TRAP_PF, regs);
+	if (!running_inband())
+		return;
+
 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 
 	if (hw_error_code & X86_PF_WRITE)
@@ -1307,7 +1384,7 @@ void do_user_addr_fault(struct pt_regs *regs,
 	 */
 	if (is_vsyscall_vaddr(address)) {
 		if (emulate_vsyscall(hw_error_code, regs, address))
-			return;
+			goto out;
 	}
 #endif
 
@@ -1340,7 +1417,7 @@ void do_user_addr_fault(struct pt_regs *regs,
 			 * which we do not expect faults.
 			 */
 			bad_area_nosemaphore(regs, hw_error_code, address);
-			return;
+			goto out;
 		}
 retry:
 		mmap_read_lock(mm);
@@ -1357,17 +1434,17 @@ void do_user_addr_fault(struct pt_regs *regs,
 		vma = find_vma(mm, address);
 	if (unlikely(!vma)) {
 		bad_area(regs, hw_error_code, address);
-		return;
+		goto out;
 	}
 	if (likely(vma->vm_start <= address))
 		goto good_area;
 	if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
 		bad_area(regs, hw_error_code, address);
-		return;
+		goto out;
 	}
 	if (unlikely(expand_stack(vma, address))) {
 		bad_area(regs, hw_error_code, address);
-		return;
+		goto out;
 	}
 
 	/*
@@ -1377,7 +1454,7 @@ void do_user_addr_fault(struct pt_regs *regs,
 good_area:
 	if (unlikely(access_error(hw_error_code, vma))) {
 		bad_area_access_error(regs, hw_error_code, address, vma);
-		return;
+		goto out;
 	}
 
 	/*
@@ -1400,7 +1477,7 @@ void do_user_addr_fault(struct pt_regs *regs,
 		if (!user_mode(regs))
 			no_context(regs, hw_error_code, address, SIGBUS,
 				   BUS_ADRERR);
-		return;
+		goto out;
 	}
 
 	/*
@@ -1426,10 +1503,12 @@ void do_user_addr_fault(struct pt_regs *regs,
 done:
 	if (unlikely(fault & VM_FAULT_ERROR)) {
 		mm_fault_error(regs, hw_error_code, address, fault);
-		return;
+		goto out;
 	}
 
 	check_v8086_mode(regs, address, tsk);
+out:
+	oob_trap_unwind(X86_TRAP_PF, regs);
 }
 NOKPROBE_SYMBOL(do_user_addr_fault);
 
@@ -1448,7 +1527,8 @@ trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code,
 
 static __always_inline void
 handle_page_fault(struct pt_regs *regs, unsigned long error_code,
-			      unsigned long address)
+		unsigned long address,
+		irqentry_state_t state)
 {
 	trace_page_fault_entries(regs, error_code, address);
 
@@ -1459,7 +1539,7 @@ handle_page_fault(struct pt_regs *regs, unsigned long error_code,
 	if (unlikely(fault_in_kernel_space(address))) {
 		do_kern_addr_fault(regs, error_code, address);
 	} else {
-		do_user_addr_fault(regs, error_code, address);
+		do_user_addr_fault(regs, error_code, address, state);
 		/*
 		 * User address page fault handling might have reenabled
 		 * interrupts. Fixing up all potential exit points of
@@ -1467,7 +1547,7 @@ handle_page_fault(struct pt_regs *regs, unsigned long error_code,
 		 * doable w/o creating an unholy mess or turning the code
 		 * upside down.
 		 */
-		local_irq_disable();
+		cond_disable_irqs();
 	}
 }
 
@@ -1515,8 +1595,46 @@ DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault)
 	state = irqentry_enter(regs);
 
 	instrumentation_begin();
-	handle_page_fault(regs, error_code, address);
+	handle_page_fault(regs, error_code, address, state);
 	instrumentation_end();
 
 	irqentry_exit(regs, state);
 }
+
+#ifdef CONFIG_DOVETAIL
+
+void arch_advertise_page_mapping(unsigned long start, unsigned long end)
+{
+	unsigned long next, addr = start;
+	pgd_t *pgd, *pgd_ref;
+	struct page *page;
+
+	/*
+	 * APEI may create temporary mappings in interrupt context -
+	 * nothing we can and need to propagate globally.
+	 */
+	if (in_interrupt())
+		return;
+
+	if (!(start >= VMALLOC_START && start < VMALLOC_END))
+		return;
+
+	do {
+		next = pgd_addr_end(addr, end);
+		pgd_ref = pgd_offset_k(addr);
+		if (pgd_none(*pgd_ref))
+			continue;
+		spin_lock(&pgd_lock);
+		list_for_each_entry(page, &pgd_list, lru) {
+			pgd = page_address(page) + pgd_index(addr);
+			if (pgd_none(*pgd))
+				set_pgd(pgd, *pgd_ref);
+		}
+		spin_unlock(&pgd_lock);
+		addr = next;
+	} while (addr != end);
+
+	arch_flush_lazy_mmu_mode();
+}
+
+#endif
diff --git a/kernel/arch/x86/mm/tlb.c b/kernel/arch/x86/mm/tlb.c
index 569ac1d57f..b720da2bc1 100644
--- a/kernel/arch/x86/mm/tlb.c
+++ b/kernel/arch/x86/mm/tlb.c
@@ -5,6 +5,7 @@
 #include <linux/spinlock.h>
 #include <linux/smp.h>
 #include <linux/interrupt.h>
+#include <linux/irq_pipeline.h>
 #include <linux/export.h>
 #include <linux/cpu.h>
 #include <linux/debugfs.h>
@@ -309,10 +310,12 @@ EXPORT_SYMBOL_GPL(leave_mm);
 void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	       struct task_struct *tsk)
 {
-	unsigned long flags;
+	unsigned long flags, _flags;
 
 	local_irq_save(flags);
+	protect_inband_mm(_flags);
 	switch_mm_irqs_off(prev, next, tsk);
+	unprotect_inband_mm(_flags);
 	local_irq_restore(flags);
 }
 
@@ -440,7 +443,9 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 	 */
 
 	/* We don't want flush_tlb_func_* to run concurrently with us. */
-	if (IS_ENABLED(CONFIG_PROVE_LOCKING))
+	if (IS_ENABLED(CONFIG_DOVETAIL))
+		WARN_ON_ONCE(!hard_irqs_disabled());
+	else if (IS_ENABLED(CONFIG_PROVE_LOCKING))
 		WARN_ON_ONCE(!irqs_disabled());
 
 	/*
@@ -666,15 +671,24 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
 	 *                   wants us to catch up to.
 	 */
 	struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
-	u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
-	u64 mm_tlb_gen = atomic64_read(&loaded_mm->context.tlb_gen);
-	u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen);
+	u32 loaded_mm_asid;
+	u64 mm_tlb_gen;
+	u64 local_tlb_gen;
+	unsigned long flags;
 
 	/* This code cannot presently handle being reentered. */
 	VM_WARN_ON(!irqs_disabled());
 
-	if (unlikely(loaded_mm == &init_mm))
+	protect_inband_mm(flags);
+
+	loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
+	mm_tlb_gen = atomic64_read(&loaded_mm->context.tlb_gen);
+	local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen);
+
+	if (unlikely(loaded_mm == &init_mm)) {
+		unprotect_inband_mm(flags);
 		return;
+	}
 
 	VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].ctx_id) !=
 		   loaded_mm->context.ctx_id);
@@ -690,6 +704,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
 		 * IPIs to lazy TLB mode CPUs.
 		 */
 		switch_mm_irqs_off(NULL, &init_mm, NULL);
+		unprotect_inband_mm(flags);
 		return;
 	}
 
@@ -700,6 +715,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
 		 * be handled can catch us all the way up, leaving no work for
 		 * the second flush.
 		 */
+		unprotect_inband_mm(flags);
 		trace_tlb_flush(reason, 0);
 		return;
 	}
@@ -707,6 +723,8 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
 	WARN_ON_ONCE(local_tlb_gen > mm_tlb_gen);
 	WARN_ON_ONCE(f->new_tlb_gen > mm_tlb_gen);
 
+	unprotect_inband_mm(flags);
+
 	/*
 	 * If we get to this point, we know that our TLB is out of date.
 	 * This does not strictly imply that we need to flush (it's
@@ -1063,7 +1081,7 @@ STATIC_NOPV void native_flush_tlb_global(void)
 	 * from interrupts. (Use the raw variant because this code can
 	 * be called from deep inside debugging code.)
 	 */
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	cr4 = this_cpu_read(cpu_tlbstate.cr4);
 	/* toggle PGE */
@@ -1071,7 +1089,7 @@ STATIC_NOPV void native_flush_tlb_global(void)
 	/* write old PGE again and flush TLBs */
 	native_write_cr4(cr4);
 
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 /*
@@ -1079,6 +1097,8 @@ STATIC_NOPV void native_flush_tlb_global(void)
  */
 STATIC_NOPV void native_flush_tlb_local(void)
 {
+	unsigned long flags;
+
 	/*
 	 * Preemption or interrupts must be disabled to protect the access
 	 * to the per CPU variable and to prevent being preempted between
@@ -1086,10 +1106,14 @@ STATIC_NOPV void native_flush_tlb_local(void)
 	 */
 	WARN_ON_ONCE(preemptible());
 
+	flags = hard_cond_local_irq_save();
+
 	invalidate_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));
 
 	/* If current->mm == NULL then the read_cr3() "borrows" an mm */
 	native_write_cr3(__native_read_cr3());
+
+	hard_cond_local_irq_restore(flags);
 }
 
 void flush_tlb_local(void)
@@ -1164,6 +1188,16 @@ bool nmi_uaccess_okay(void)
 
 	VM_WARN_ON_ONCE(!loaded_mm);
 
+	/*
+	 * There would be no way for the companion core to switch an
+	 * out-of-band task back in-band in order to handle an access
+	 * fault over NMI safely. Tell the caller that uaccess from
+	 * NMI is NOT ok if the preempted task was running
+	 * out-of-band.
+	 */
+	if (running_oob())
+		return false;
+
 	/*
 	 * The condition we want to check is
 	 * current_mm->pgd == __va(read_cr3_pa()).  This may be slow, though,
diff --git a/kernel/arch/x86/xen/Kconfig b/kernel/arch/x86/xen/Kconfig
index 218acbd5c7..a02524c411 100644
--- a/kernel/arch/x86/xen/Kconfig
+++ b/kernel/arch/x86/xen/Kconfig
@@ -5,7 +5,7 @@
 
 config XEN
 	bool "Xen guest support"
-	depends on PARAVIRT
+	depends on PARAVIRT && !IRQ_PIPELINE
 	select PARAVIRT_CLOCK
 	select X86_HV_CALLBACK_VECTOR
 	depends on X86_64 || (X86_32 && X86_PAE)
diff --git a/kernel/arch/x86/xen/enlighten_hvm.c b/kernel/arch/x86/xen/enlighten_hvm.c
index ec50b7423a..8617927d0b 100644
--- a/kernel/arch/x86/xen/enlighten_hvm.c
+++ b/kernel/arch/x86/xen/enlighten_hvm.c
@@ -120,7 +120,8 @@ static void __init init_hvm_pv_info(void)
 		this_cpu_write(xen_vcpu_id, smp_processor_id());
 }
 
-DEFINE_IDTENTRY_SYSVEC(sysvec_xen_hvm_callback)
+DEFINE_IDTENTRY_SYSVEC_PIPELINED(HYPERVISOR_CALLBACK_VECTOR,
+				 sysvec_xen_hvm_callback)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
diff --git a/kernel/drivers/base/regmap/internal.h b/kernel/drivers/base/regmap/internal.h
index 0097696c31..ec9bea3e00 100644
--- a/kernel/drivers/base/regmap/internal.h
+++ b/kernel/drivers/base/regmap/internal.h
@@ -50,7 +50,10 @@ struct regmap {
 	union {
 		struct mutex mutex;
 		struct {
-			spinlock_t spinlock;
+			union {
+				spinlock_t spinlock;
+				hard_spinlock_t oob_lock;
+			};
 			unsigned long spinlock_flags;
 		};
 	};
diff --git a/kernel/drivers/base/regmap/regmap-irq.c b/kernel/drivers/base/regmap/regmap-irq.c
index 4466f8bdab..ea1fc2ff2c 100644
--- a/kernel/drivers/base/regmap/regmap-irq.c
+++ b/kernel/drivers/base/regmap/regmap-irq.c
@@ -331,6 +331,7 @@ static const struct irq_chip regmap_irq_chip = {
 	.irq_enable		= regmap_irq_enable,
 	.irq_set_type		= regmap_irq_set_type,
 	.irq_set_wake		= regmap_irq_set_wake,
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static inline int read_sub_irq_data(struct regmap_irq_chip_data *data,
diff --git a/kernel/drivers/base/regmap/regmap.c b/kernel/drivers/base/regmap/regmap.c
index 55a30afc14..f586293892 100644
--- a/kernel/drivers/base/regmap/regmap.c
+++ b/kernel/drivers/base/regmap/regmap.c
@@ -14,6 +14,7 @@
 #include <linux/property.h>
 #include <linux/rbtree.h>
 #include <linux/sched.h>
+#include <linux/dovetail.h>
 #include <linux/delay.h>
 #include <linux/log2.h>
 #include <linux/hwspinlock.h>
@@ -523,6 +524,23 @@ __releases(&map->spinlock)
 	spin_unlock_irqrestore(&map->spinlock, map->spinlock_flags);
 }
 
+static void regmap_lock_oob(void *__map)
+__acquires(&map->oob_lock)
+{
+	struct regmap *map = __map;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&map->oob_lock, flags);
+	map->spinlock_flags = flags;
+}
+
+static void regmap_unlock_oob(void *__map)
+__releases(&map->oob_lock)
+{
+	struct regmap *map = __map;
+	raw_spin_unlock_irqrestore(&map->oob_lock, map->spinlock_flags);
+}
+
 static void dev_get_regmap_release(struct device *dev, void *res)
 {
 	/*
@@ -761,18 +779,29 @@ struct regmap *__regmap_init(struct device *dev,
 	} else {
 		if ((bus && bus->fast_io) ||
 		    config->fast_io) {
-			spin_lock_init(&map->spinlock);
-			map->lock = regmap_lock_spinlock;
-			map->unlock = regmap_unlock_spinlock;
-			lockdep_set_class_and_name(&map->spinlock,
-						   lock_key, lock_name);
-		} else {
+			if (dovetailing() && config->oob_io) {
+				raw_spin_lock_init(&map->oob_lock);
+				map->lock = regmap_lock_oob;
+				map->unlock = regmap_unlock_oob;
+				lockdep_set_class_and_name(&map->oob_lock,
+							lock_key, lock_name);
+			} else {
+				spin_lock_init(&map->spinlock);
+				map->lock = regmap_lock_spinlock;
+				map->unlock = regmap_unlock_spinlock;
+				lockdep_set_class_and_name(&map->spinlock,
+							lock_key, lock_name);
+			}
+		} else if (!config->oob_io) {
 			mutex_init(&map->mutex);
 			map->lock = regmap_lock_mutex;
 			map->unlock = regmap_unlock_mutex;
 			map->can_sleep = true;
 			lockdep_set_class_and_name(&map->mutex,
 						   lock_key, lock_name);
+		} else {
+			ret = -ENXIO;
+			goto err_name;
 		}
 		map->lock_arg = map;
 	}
diff --git a/kernel/drivers/clocksource/Kconfig b/kernel/drivers/clocksource/Kconfig
index 99c6b44cb4..ae971d3d17 100644
--- a/kernel/drivers/clocksource/Kconfig
+++ b/kernel/drivers/clocksource/Kconfig
@@ -25,6 +25,7 @@ config I8253_LOCK
 config OMAP_DM_TIMER
 	bool
 	select TIMER_OF
+	imply GENERIC_CLOCKSOURCE_VDSO
 
 config CLKBLD_I8253
 	def_bool y if CLKSRC_I8253 || CLKEVT_I8253 || I8253_LOCK
@@ -58,6 +59,8 @@ config DIGICOLOR_TIMER
 
 config DW_APB_TIMER
 	bool "DW APB timer driver" if COMPILE_TEST
+        select CLKSRC_MMIO
+	imply GENERIC_CLOCKSOURCE_VDSO
 	help
 	  Enables the support for the dw_apb timer.
 
@@ -395,6 +398,7 @@ config SUN50I_ERRATUM_UNKNOWN1
 config ARM_GLOBAL_TIMER
 	bool "Support for the ARM global timer" if COMPILE_TEST
 	select TIMER_OF if OF
+	imply GENERIC_CLOCKSOURCE_VDSO
 	depends on ARM
 	help
 	  This option enables support for the ARM global timer unit.
@@ -444,6 +448,7 @@ config ATMEL_TCB_CLKSRC
 config CLKSRC_EXYNOS_MCT
 	bool "Exynos multi core timer driver" if COMPILE_TEST
 	depends on ARM || ARM64
+	imply GENERIC_CLOCKSOURCE_VDSO
 	help
 	  Support for Multi Core Timer controller on Exynos SoCs.
 
@@ -620,7 +625,7 @@ config H8300_TPU
 config CLKSRC_IMX_GPT
 	bool "Clocksource using i.MX GPT" if COMPILE_TEST
 	depends on (ARM || ARM64) && CLKDEV_LOOKUP
-	select CLKSRC_MMIO
+	imply GENERIC_CLOCKSOURCE_VDSO
 
 config CLKSRC_IMX_TPM
 	bool "Clocksource using i.MX TPM" if COMPILE_TEST
@@ -642,7 +647,7 @@ config CLKSRC_ST_LPC
 	bool "Low power clocksource found in the LPC" if COMPILE_TEST
 	select TIMER_OF if OF
 	depends on HAS_IOMEM
-	select CLKSRC_MMIO
+	imply GENERIC_CLOCKSOURCE_VDSO
 	help
 	  Enable this option to use the Low Power controller timer
 	  as clocksource.
diff --git a/kernel/drivers/clocksource/arm_arch_timer.c b/kernel/drivers/clocksource/arm_arch_timer.c
index f4881764bf..fb4fb16f4a 100644
--- a/kernel/drivers/clocksource/arm_arch_timer.c
+++ b/kernel/drivers/clocksource/arm_arch_timer.c
@@ -21,6 +21,7 @@
 #include <linux/of_address.h>
 #include <linux/io.h>
 #include <linux/slab.h>
+#include <linux/dovetail.h>
 #include <linux/sched/clock.h>
 #include <linux/sched_clock.h>
 #include <linux/acpi.h>
@@ -644,7 +645,7 @@ static __always_inline irqreturn_t timer_handler(const int access,
 	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
-		evt->event_handler(evt);
+		clockevents_handle_event(evt);
 		return IRQ_HANDLED;
 	}
 
@@ -753,7 +754,7 @@ static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 static void __arch_timer_setup(unsigned type,
 			       struct clock_event_device *clk)
 {
-	clk->features = CLOCK_EVT_FEAT_ONESHOT;
+	clk->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PIPELINE;
 
 	if (type == ARCH_TIMER_TYPE_CP15) {
 		typeof(clk->set_next_event) sne;
@@ -864,6 +865,9 @@ static void arch_counter_set_user_access(void)
 	else
 		cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 
+	if (IS_ENABLED(CONFIG_GENERIC_CLOCKSOURCE_VDSO))
+		cntkctl |= ARCH_TIMER_USR_PT_ACCESS_EN;
+
 	arch_timer_set_cntkctl(cntkctl);
 }
 
@@ -897,6 +901,7 @@ static int arch_timer_starting_cpu(unsigned int cpu)
 	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 
 	if (arch_timer_has_nonsecure_ppi()) {
+		clk->irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
 		flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 		enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
 				  flags);
@@ -1015,6 +1020,8 @@ static void __init arch_counter_register(unsigned type)
 
 		arch_timer_read_counter = rd;
 		clocksource_counter.vdso_clock_mode = vdso_default;
+		if (vdso_default != VDSO_CLOCKMODE_NONE)
+			clocksource_counter.vdso_type = CLOCKSOURCE_VDSO_ARCHITECTED;
 	} else {
 		arch_timer_read_counter = arch_counter_get_cntvct_mem;
 	}
diff --git a/kernel/drivers/clocksource/arm_global_timer.c b/kernel/drivers/clocksource/arm_global_timer.c
index 88b2d38a7a..3273fc6b56 100644
--- a/kernel/drivers/clocksource/arm_global_timer.c
+++ b/kernel/drivers/clocksource/arm_global_timer.c
@@ -153,11 +153,11 @@ static irqreturn_t gt_clockevent_interrupt(int irq, void *dev_id)
 	 *	the Global Timer flag _after_ having incremented
 	 *	the Comparator register	value to a higher value.
 	 */
-	if (clockevent_state_oneshot(evt))
+	if (clockevent_is_oob(evt) || clockevent_state_oneshot(evt))
 		gt_compare_set(ULONG_MAX, 0);
 
 	writel_relaxed(GT_INT_STATUS_EVENT_FLAG, gt_base + GT_INT_STATUS);
-	evt->event_handler(evt);
+	clockevents_handle_event(evt);
 
 	return IRQ_HANDLED;
 }
@@ -168,7 +168,7 @@ static int gt_starting_cpu(unsigned int cpu)
 
 	clk->name = "arm_global_timer";
 	clk->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT |
-		CLOCK_EVT_FEAT_PERCPU;
+		CLOCK_EVT_FEAT_PERCPU | CLOCK_EVT_FEAT_PIPELINE;
 	clk->set_state_shutdown = gt_clockevent_shutdown;
 	clk->set_state_periodic = gt_clockevent_set_periodic;
 	clk->set_state_oneshot = gt_clockevent_shutdown;
@@ -192,11 +192,6 @@ static int gt_dying_cpu(unsigned int cpu)
 	return 0;
 }
 
-static u64 gt_clocksource_read(struct clocksource *cs)
-{
-	return gt_counter_read();
-}
-
 static void gt_resume(struct clocksource *cs)
 {
 	unsigned long ctrl;
@@ -207,13 +202,15 @@ static void gt_resume(struct clocksource *cs)
 		writel(GT_CONTROL_TIMER_ENABLE, gt_base + GT_CONTROL);
 }
 
-static struct clocksource gt_clocksource = {
-	.name	= "arm_global_timer",
-	.rating	= 300,
-	.read	= gt_clocksource_read,
-	.mask	= CLOCKSOURCE_MASK(64),
-	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
-	.resume = gt_resume,
+static struct clocksource_user_mmio gt_clocksource = {
+	.mmio.clksrc = {
+		.name	= "arm_global_timer",
+		.rating	= 300,
+		.read	= clocksource_dual_mmio_readl_up,
+		.mask	= CLOCKSOURCE_MASK(64),
+		.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
+		.resume = gt_resume,
+	},
 };
 
 #ifdef CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK
@@ -240,6 +237,8 @@ static void __init gt_delay_timer_init(void)
 
 static int __init gt_clocksource_init(void)
 {
+	struct clocksource_mmio_regs mmr;
+
 	writel(0, gt_base + GT_CONTROL);
 	writel(0, gt_base + GT_COUNTER0);
 	writel(0, gt_base + GT_COUNTER1);
@@ -249,7 +248,13 @@ static int __init gt_clocksource_init(void)
 #ifdef CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK
 	sched_clock_register(gt_sched_clock_read, 64, gt_clk_rate);
 #endif
-	return clocksource_register_hz(&gt_clocksource, gt_clk_rate);
+	mmr.reg_upper = gt_base + GT_COUNTER1;
+	mmr.reg_lower = gt_base + GT_COUNTER0;
+	mmr.bits_upper = 32;
+	mmr.bits_lower = 32;
+	mmr.revmap = NULL;
+
+	return clocksource_user_mmio_init(&gt_clocksource, &mmr, gt_clk_rate);
 }
 
 static int __init global_timer_of_register(struct device_node *np)
@@ -299,8 +304,8 @@ static int __init global_timer_of_register(struct device_node *np)
 		goto out_clk;
 	}
 
-	err = request_percpu_irq(gt_ppi, gt_clockevent_interrupt,
-				 "gt", gt_evt);
+	err = __request_percpu_irq(gt_ppi, gt_clockevent_interrupt,
+				   IRQF_TIMER, "gt", gt_evt);
 	if (err) {
 		pr_warn("global-timer: can't register interrupt %d (%d)\n",
 			gt_ppi, err);
diff --git a/kernel/drivers/clocksource/bcm2835_timer.c b/kernel/drivers/clocksource/bcm2835_timer.c
index 1592650b2c..687e9f2b4e 100644
--- a/kernel/drivers/clocksource/bcm2835_timer.c
+++ b/kernel/drivers/clocksource/bcm2835_timer.c
@@ -53,25 +53,33 @@ static int bcm2835_time_set_next_event(unsigned long event,
 static irqreturn_t bcm2835_time_interrupt(int irq, void *dev_id)
 {
 	struct bcm2835_timer *timer = dev_id;
-	void (*event_handler)(struct clock_event_device *);
+
 	if (readl_relaxed(timer->control) & timer->match_mask) {
 		writel_relaxed(timer->match_mask, timer->control);
 
-		event_handler = READ_ONCE(timer->evt.event_handler);
-		if (event_handler)
-			event_handler(&timer->evt);
+		clockevents_handle_event(&timer->evt);
 		return IRQ_HANDLED;
 	} else {
 		return IRQ_NONE;
 	}
 }
 
+static struct clocksource_user_mmio clocksource_bcm2835 = {
+	.mmio.clksrc = {
+		.rating		= 300,
+		.read		= clocksource_mmio_readl_up,
+		.mask		= CLOCKSOURCE_MASK(32),
+		.flags		= CLOCK_SOURCE_IS_CONTINUOUS,
+	},
+};
+
 static int __init bcm2835_timer_init(struct device_node *node)
 {
 	void __iomem *base;
 	u32 freq;
 	int irq, ret;
 	struct bcm2835_timer *timer;
+	struct clocksource_mmio_regs mmr;
 
 	base = of_iomap(node, 0);
 	if (!base) {
@@ -88,8 +96,13 @@ static int __init bcm2835_timer_init(struct device_node *node)
 	system_clock = base + REG_COUNTER_LO;
 	sched_clock_register(bcm2835_sched_read, 32, freq);
 
-	clocksource_mmio_init(base + REG_COUNTER_LO, node->name,
-		freq, 300, 32, clocksource_mmio_readl_up);
+	mmr.reg_lower = base + REG_COUNTER_LO;
+	mmr.bits_lower = 32;
+	mmr.reg_upper = 0;
+	mmr.bits_upper = 0;
+	mmr.revmap = NULL;
+	clocksource_bcm2835.mmio.clksrc.name = node->name;
+	clocksource_user_mmio_init(&clocksource_bcm2835, &mmr, freq);
 
 	irq = irq_of_parse_and_map(node, DEFAULT_TIMER);
 	if (irq <= 0) {
@@ -109,7 +122,7 @@ static int __init bcm2835_timer_init(struct device_node *node)
 	timer->match_mask = BIT(DEFAULT_TIMER);
 	timer->evt.name = node->name;
 	timer->evt.rating = 300;
-	timer->evt.features = CLOCK_EVT_FEAT_ONESHOT;
+	timer->evt.features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PIPELINE;
 	timer->evt.set_next_event = bcm2835_time_set_next_event;
 	timer->evt.cpumask = cpumask_of(0);
 
diff --git a/kernel/drivers/clocksource/clksrc_st_lpc.c b/kernel/drivers/clocksource/clksrc_st_lpc.c
index 419a886876..b30b814d27 100644
--- a/kernel/drivers/clocksource/clksrc_st_lpc.c
+++ b/kernel/drivers/clocksource/clksrc_st_lpc.c
@@ -51,7 +51,7 @@ static int __init st_clksrc_init(void)
 
 	sched_clock_register(st_clksrc_sched_clock_read, 32, rate);
 
-	ret = clocksource_mmio_init(ddata.base + LPC_LPT_LSB_OFF,
+	ret = clocksource_user_single_mmio_init(ddata.base + LPC_LPT_LSB_OFF,
 				    "clksrc-st-lpc", rate, 300, 32,
 				    clocksource_mmio_readl_up);
 	if (ret) {
diff --git a/kernel/drivers/clocksource/dw_apb_timer.c b/kernel/drivers/clocksource/dw_apb_timer.c
index f5f24a95ee..a974b9df44 100644
--- a/kernel/drivers/clocksource/dw_apb_timer.c
+++ b/kernel/drivers/clocksource/dw_apb_timer.c
@@ -43,7 +43,7 @@ ced_to_dw_apb_ced(struct clock_event_device *evt)
 static inline struct dw_apb_clocksource *
 clocksource_to_dw_apb_clocksource(struct clocksource *cs)
 {
-	return container_of(cs, struct dw_apb_clocksource, cs);
+	return container_of(cs, struct dw_apb_clocksource, ummio.mmio.clksrc);
 }
 
 static inline u32 apbt_readl(struct dw_apb_timer *timer, unsigned long offs)
@@ -343,18 +343,6 @@ void dw_apb_clocksource_start(struct dw_apb_clocksource *dw_cs)
 	dw_apb_clocksource_read(dw_cs);
 }
 
-static u64 __apbt_read_clocksource(struct clocksource *cs)
-{
-	u32 current_count;
-	struct dw_apb_clocksource *dw_cs =
-		clocksource_to_dw_apb_clocksource(cs);
-
-	current_count = apbt_readl_relaxed(&dw_cs->timer,
-					APBTMR_N_CURRENT_VALUE);
-
-	return (u64)~current_count;
-}
-
 static void apbt_restart_clocksource(struct clocksource *cs)
 {
 	struct dw_apb_clocksource *dw_cs =
@@ -376,7 +364,7 @@ static void apbt_restart_clocksource(struct clocksource *cs)
  * dw_apb_clocksource_register() as the next step.
  */
 struct dw_apb_clocksource *
-dw_apb_clocksource_init(unsigned rating, const char *name, void __iomem *base,
+__init dw_apb_clocksource_init(unsigned rating, const char *name, void __iomem *base,
 			unsigned long freq)
 {
 	struct dw_apb_clocksource *dw_cs = kzalloc(sizeof(*dw_cs), GFP_KERNEL);
@@ -386,12 +374,12 @@ dw_apb_clocksource_init(unsigned rating, const char *name, void __iomem *base,
 
 	dw_cs->timer.base = base;
 	dw_cs->timer.freq = freq;
-	dw_cs->cs.name = name;
-	dw_cs->cs.rating = rating;
-	dw_cs->cs.read = __apbt_read_clocksource;
-	dw_cs->cs.mask = CLOCKSOURCE_MASK(32);
-	dw_cs->cs.flags = CLOCK_SOURCE_IS_CONTINUOUS;
-	dw_cs->cs.resume = apbt_restart_clocksource;
+	dw_cs->ummio.mmio.clksrc.name = name;
+	dw_cs->ummio.mmio.clksrc.rating = rating;
+	dw_cs->ummio.mmio.clksrc.read = clocksource_mmio_readl_down;
+	dw_cs->ummio.mmio.clksrc.mask = CLOCKSOURCE_MASK(32);
+	dw_cs->ummio.mmio.clksrc.flags = CLOCK_SOURCE_IS_CONTINUOUS;
+	dw_cs->ummio.mmio.clksrc.resume = apbt_restart_clocksource;
 
 	return dw_cs;
 }
@@ -401,9 +389,17 @@ dw_apb_clocksource_init(unsigned rating, const char *name, void __iomem *base,
  *
  * @dw_cs:	The clocksource to register.
  */
-void dw_apb_clocksource_register(struct dw_apb_clocksource *dw_cs)
+void __init dw_apb_clocksource_register(struct dw_apb_clocksource *dw_cs)
 {
-	clocksource_register_hz(&dw_cs->cs, dw_cs->timer.freq);
+	struct clocksource_mmio_regs mmr;
+
+	mmr.reg_lower = dw_cs->timer.base + APBTMR_N_CURRENT_VALUE;
+	mmr.bits_lower = 32;
+	mmr.reg_upper = 0;
+	mmr.bits_upper = 0;
+	mmr.revmap = NULL;
+
+	clocksource_user_mmio_init(&dw_cs->ummio, &mmr, dw_cs->timer.freq);
 }
 
 /**
diff --git a/kernel/drivers/clocksource/exynos_mct.c b/kernel/drivers/clocksource/exynos_mct.c
index df194b05e9..243adda8c1 100644
--- a/kernel/drivers/clocksource/exynos_mct.c
+++ b/kernel/drivers/clocksource/exynos_mct.c
@@ -194,23 +194,20 @@ static u32 notrace exynos4_read_count_32(void)
 	return readl_relaxed(reg_base + EXYNOS4_MCT_G_CNT_L);
 }
 
-static u64 exynos4_frc_read(struct clocksource *cs)
-{
-	return exynos4_read_count_32();
-}
-
 static void exynos4_frc_resume(struct clocksource *cs)
 {
 	exynos4_mct_frc_start();
 }
 
-static struct clocksource mct_frc = {
-	.name		= "mct-frc",
-	.rating		= 450,	/* use value higher than ARM arch timer */
-	.read		= exynos4_frc_read,
-	.mask		= CLOCKSOURCE_MASK(32),
-	.flags		= CLOCK_SOURCE_IS_CONTINUOUS,
-	.resume		= exynos4_frc_resume,
+static struct clocksource_user_mmio mct_frc = {
+	.mmio.clksrc = {
+		.name		= "mct-frc",
+		.rating		= 450,	/* use value higher than ARM arch timer */
+		.read		= clocksource_mmio_readl_up,
+		.mask		= CLOCKSOURCE_MASK(32),
+		.flags		= CLOCK_SOURCE_IS_CONTINUOUS,
+		.resume		= exynos4_frc_resume,
+	},
 };
 
 static u64 notrace exynos4_read_sched_clock(void)
@@ -231,6 +228,8 @@ static cycles_t exynos4_read_current_timer(void)
 
 static int __init exynos4_clocksource_init(void)
 {
+	struct clocksource_mmio_regs mmr;
+
 	exynos4_mct_frc_start();
 
 #if defined(CONFIG_ARM)
@@ -239,8 +238,13 @@ static int __init exynos4_clocksource_init(void)
 	register_current_timer_delay(&exynos4_delay_timer);
 #endif
 
-	if (clocksource_register_hz(&mct_frc, clk_rate))
-		panic("%s: can't register clocksource\n", mct_frc.name);
+	mmr.reg_upper = NULL;
+	mmr.reg_lower = reg_base + EXYNOS4_MCT_G_CNT_L;
+	mmr.bits_upper = 0;
+	mmr.bits_lower = 32;
+	mmr.revmap = NULL;
+	if (clocksource_user_mmio_init(&mct_frc, &mmr, clk_rate))
+		panic("%s: can't register clocksource\n", mct_frc.mmio.clksrc.name);
 
 	sched_clock_register(exynos4_read_sched_clock, 32, clk_rate);
 
@@ -308,7 +312,8 @@ static int mct_set_state_periodic(struct clock_event_device *evt)
 static struct clock_event_device mct_comp_device = {
 	.name			= "mct-comp",
 	.features		= CLOCK_EVT_FEAT_PERIODIC |
-				  CLOCK_EVT_FEAT_ONESHOT,
+				  CLOCK_EVT_FEAT_ONESHOT |
+				  CLOCK_EVT_FEAT_PIPELINE,
 	.rating			= 250,
 	.set_next_event		= exynos4_comp_set_next_event,
 	.set_state_periodic	= mct_set_state_periodic,
@@ -324,7 +329,7 @@ static irqreturn_t exynos4_mct_comp_isr(int irq, void *dev_id)
 
 	exynos4_mct_write(0x1, EXYNOS4_MCT_G_INT_CSTAT);
 
-	evt->event_handler(evt);
+	clockevents_handle_event(evt);
 
 	return IRQ_HANDLED;
 }
@@ -335,7 +340,7 @@ static int exynos4_clockevent_init(void)
 	clockevents_config_and_register(&mct_comp_device, clk_rate,
 					0xf, 0xffffffff);
 	if (request_irq(mct_irqs[MCT_G0_IRQ], exynos4_mct_comp_isr,
-			IRQF_TIMER | IRQF_IRQPOLL, "mct_comp_irq",
+			IRQF_TIMER | IRQF_IRQPOLL | IRQF_OOB, "mct_comp_irq",
 			&mct_comp_device))
 		pr_err("%s: request_irq() failed\n", "mct_comp_irq");
 
@@ -434,7 +439,7 @@ static irqreturn_t exynos4_mct_tick_isr(int irq, void *dev_id)
 
 	exynos4_mct_tick_clear(mevt);
 
-	evt->event_handler(evt);
+	clockevents_handle_event(evt);
 
 	return IRQ_HANDLED;
 }
@@ -456,7 +461,8 @@ static int exynos4_mct_starting_cpu(unsigned int cpu)
 	evt->set_state_oneshot = set_state_shutdown;
 	evt->set_state_oneshot_stopped = set_state_shutdown;
 	evt->tick_resume = set_state_shutdown;
-	evt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT;
+	evt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | \
+		CLOCK_EVT_FEAT_PIPELINE;
 	evt->rating = 500;	/* use value higher than ARM arch timer */
 
 	exynos4_mct_write(TICK_BASE_CNT, mevt->base + MCT_L_TCNTB_OFFSET);
@@ -541,9 +547,9 @@ static int __init exynos4_timer_interrupts(struct device_node *np,
 
 	if (mct_int_type == MCT_INT_PPI) {
 
-		err = request_percpu_irq(mct_irqs[MCT_L0_IRQ],
-					 exynos4_mct_tick_isr, "MCT",
-					 &percpu_mct_tick);
+		err = __request_percpu_irq(mct_irqs[MCT_L0_IRQ],
+					exynos4_mct_tick_isr, IRQF_TIMER,
+					"MCT", &percpu_mct_tick);
 		WARN(err, "MCT: can't request IRQ %d (%d)\n",
 		     mct_irqs[MCT_L0_IRQ], err);
 	} else {
diff --git a/kernel/drivers/clocksource/mmio.c b/kernel/drivers/clocksource/mmio.c
index 826dcc4262..163a50a7fa 100644
--- a/kernel/drivers/clocksource/mmio.c
+++ b/kernel/drivers/clocksource/mmio.c
@@ -6,12 +6,31 @@
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/uaccess.h>
+#include <linux/miscdevice.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/device.h>
 
-struct clocksource_mmio {
-	void __iomem *reg;
-	struct clocksource clksrc;
+struct clocksource_user_mapping {
+	struct mm_struct *mm;
+	struct clocksource_user_mmio *ucs;
+	void *regs;
+	struct hlist_node link;
+	atomic_t refs;
 };
 
+static struct class *user_mmio_class;
+static dev_t user_mmio_devt;
+
+static DEFINE_SPINLOCK(user_clksrcs_lock);
+static unsigned int user_clksrcs_count;
+static LIST_HEAD(user_clksrcs);
+
 static inline struct clocksource_mmio *to_mmio_clksrc(struct clocksource *c)
 {
 	return container_of(c, struct clocksource_mmio, clksrc);
@@ -38,6 +57,53 @@ u64 clocksource_mmio_readw_down(struct clocksource *c)
 	return ~(u64)readw_relaxed(to_mmio_clksrc(c)->reg) & c->mask;
 }
 
+static inline struct clocksource_user_mmio *
+to_mmio_ucs(struct clocksource *c)
+{
+	return container_of(c, struct clocksource_user_mmio, mmio.clksrc);
+}
+
+u64 clocksource_dual_mmio_readl_up(struct clocksource *c)
+{
+	struct clocksource_user_mmio *ucs = to_mmio_ucs(c);
+	u32 upper, old_upper, lower;
+
+	upper = readl_relaxed(ucs->reg_upper);
+	do {
+		old_upper = upper;
+		lower = readl_relaxed(ucs->mmio.reg);
+		upper = readl_relaxed(ucs->reg_upper);
+	} while (upper != old_upper);
+
+	return (((u64)upper) << ucs->bits_lower) | lower;
+}
+
+u64 clocksource_dual_mmio_readw_up(struct clocksource *c)
+{
+	struct clocksource_user_mmio *ucs = to_mmio_ucs(c);
+	u16 upper, old_upper, lower;
+
+	upper = readw_relaxed(ucs->reg_upper);
+	do {
+		old_upper = upper;
+		lower = readw_relaxed(ucs->mmio.reg);
+		upper = readw_relaxed(ucs->reg_upper);
+	} while (upper != old_upper);
+
+	return (((u64)upper) << ucs->bits_lower) | lower;
+}
+
+static void mmio_base_init(const char *name,int rating, unsigned int bits,
+			   u64 (*read)(struct clocksource *),
+			   struct clocksource *cs)
+{
+	cs->name = name;
+	cs->rating = rating;
+	cs->read = read;
+	cs->mask = CLOCKSOURCE_MASK(bits);
+	cs->flags = CLOCK_SOURCE_IS_CONTINUOUS;
+}
+
 /**
  * clocksource_mmio_init - Initialize a simple mmio based clocksource
  * @base:	Virtual address of the clock readout register
@@ -52,6 +118,7 @@ int clocksource_mmio_init(void __iomem *base, const char *name,
 	u64 (*read)(struct clocksource *))
 {
 	struct clocksource_mmio *cs;
+	int err;
 
 	if (bits > 64 || bits < 16)
 		return -EINVAL;
@@ -61,12 +128,428 @@ int clocksource_mmio_init(void __iomem *base, const char *name,
 		return -ENOMEM;
 
 	cs->reg = base;
-	cs->clksrc.name = name;
-	cs->clksrc.rating = rating;
-	cs->clksrc.read = read;
-	cs->clksrc.mask = CLOCKSOURCE_MASK(bits);
-	cs->clksrc.flags = CLOCK_SOURCE_IS_CONTINUOUS;
+	mmio_base_init(name, rating, bits, read, &cs->clksrc);
+
+	err = clocksource_register_hz(&cs->clksrc, hz);
+	if (err < 0) {
+		kfree(cs);
+		return err;
+	}
+
+	return err;
+}
+
+static void mmio_ucs_vmopen(struct vm_area_struct *vma)
+{
+	struct clocksource_user_mapping *mapping, *clone;
+	struct clocksource_user_mmio *ucs;
+	unsigned long h_key;
+
+	mapping = vma->vm_private_data;
+
+	if (mapping->mm == vma->vm_mm) {
+		atomic_inc(&mapping->refs);
+	} else if (mapping->mm) {
+		/*
+		 * We must be duplicating the original mm upon fork(),
+		 * clone the parent ucs mapping struct then rehash it
+		 * on the child mm key. If we cannot get memory for
+		 * this, mitigate the issue for users by preventing a
+		 * stale parent mm from being matched later on by a
+		 * process which reused its mm_struct (h_key is based
+		 * on this struct address).
+		 */
+		clone = kmalloc(sizeof(*mapping), GFP_KERNEL);
+		if (clone == NULL) {
+			pr_alert("out-of-memory for UCS mapping!\n");
+			atomic_inc(&mapping->refs);
+			mapping->mm = NULL;
+			return;
+		}
+		ucs = mapping->ucs;
+		clone->mm = vma->vm_mm;
+		clone->ucs = ucs;
+		clone->regs = mapping->regs;
+		atomic_set(&clone->refs, 1);
+		vma->vm_private_data = clone;
+		h_key = (unsigned long)vma->vm_mm / sizeof(*vma->vm_mm);
+		spin_lock(&ucs->lock);
+		hash_add(ucs->mappings, &clone->link, h_key);
+		spin_unlock(&ucs->lock);
+	}
+}
+
+static void mmio_ucs_vmclose(struct vm_area_struct *vma)
+{
+	struct clocksource_user_mapping *mapping;
+
+	mapping = vma->vm_private_data;
+
+	if (atomic_dec_and_test(&mapping->refs)) {
+		spin_lock(&mapping->ucs->lock);
+		hash_del(&mapping->link);
+		spin_unlock(&mapping->ucs->lock);
+		kfree(mapping);
+	}
+}
+
+static const struct vm_operations_struct mmio_ucs_vmops = {
+	.open = mmio_ucs_vmopen,
+	.close = mmio_ucs_vmclose,
+};
+
+static int mmio_ucs_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	unsigned long addr, upper_pfn, lower_pfn;
+	struct clocksource_user_mapping *mapping, *tmp;
+	struct clocksource_user_mmio *ucs;
+	unsigned int bits_upper;
+	unsigned long h_key;
+	pgprot_t prot;
+	size_t pages;
+	int err;
+
+	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	if (pages > 2)
+		return -EINVAL;
+
+	vma->vm_private_data = NULL;
+
+	ucs = file->private_data;
+	upper_pfn = ucs->phys_upper >> PAGE_SHIFT;
+	lower_pfn = ucs->phys_lower >> PAGE_SHIFT;
+	bits_upper = fls(ucs->mmio.clksrc.mask) - ucs->bits_lower;
+	if (pages == 2 && (!bits_upper || upper_pfn == lower_pfn))
+		return -EINVAL;
+
+	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
+	if (!mapping)
+		return -ENOSPC;
+
+	mapping->mm = vma->vm_mm;
+	mapping->ucs = ucs;
+	mapping->regs = (void *)vma->vm_start;
+	atomic_set(&mapping->refs, 1);
+
+	vma->vm_private_data = mapping;
+	vma->vm_ops = &mmio_ucs_vmops;
+	prot = pgprot_noncached(vma->vm_page_prot);
+	addr = vma->vm_start;
+
+	err = remap_pfn_range(vma, addr, lower_pfn, PAGE_SIZE, prot);
+	if (err < 0)
+		goto fail;
+
+	if (pages > 1) {
+		addr += PAGE_SIZE;
+		err = remap_pfn_range(vma, addr, upper_pfn, PAGE_SIZE, prot);
+		if (err < 0)
+			goto fail;
+	}
+
+	h_key = (unsigned long)vma->vm_mm / sizeof(*vma->vm_mm);
+
+	spin_lock(&ucs->lock);
+	hash_for_each_possible(ucs->mappings, tmp, link, h_key) {
+		if (tmp->mm == vma->vm_mm) {
+			spin_unlock(&ucs->lock);
+			err = -EBUSY;
+			goto fail;
+		}
+	}
+	hash_add(ucs->mappings, &mapping->link, h_key);
+	spin_unlock(&ucs->lock);
+
+	return 0;
+fail:
+	kfree(mapping);
+
+	return err;
+}
+
+static long
+mmio_ucs_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct clocksource_user_mapping *mapping;
+	struct clksrc_user_mmio_info __user *u;
+	unsigned long upper_pfn, lower_pfn;
+	struct clksrc_user_mmio_info info;
+	struct clocksource_user_mmio *ucs;
+	unsigned int bits_upper;
+	void __user *map_base;
+	unsigned long h_key;
+	size_t size;
+
+	u = (struct clksrc_user_mmio_info __user *)arg;
+
+	switch (cmd) {
+	case CLKSRC_USER_MMIO_MAP:
+		break;
+	default:
+		return -ENOTTY;
+	}
+
+	h_key = (unsigned long)current->mm / sizeof(*current->mm);
+
+	ucs = file->private_data;
+	upper_pfn = ucs->phys_upper >> PAGE_SHIFT;
+	lower_pfn = ucs->phys_lower >> PAGE_SHIFT;
+	bits_upper = fls(ucs->mmio.clksrc.mask) - ucs->bits_lower;
+	size = PAGE_SIZE;
+	if (bits_upper && upper_pfn != lower_pfn)
+		size += PAGE_SIZE;
+
+	do {
+		spin_lock(&ucs->lock);
+		hash_for_each_possible(ucs->mappings, mapping, link, h_key) {
+			if (mapping->mm == current->mm) {
+				spin_unlock(&ucs->lock);
+				map_base = mapping->regs;
+				goto found;
+			}
+		}
+		spin_unlock(&ucs->lock);
+
+		map_base = (void *)
+			vm_mmap(file, 0, size, PROT_READ, MAP_SHARED, 0);
+	} while (IS_ERR(map_base) && PTR_ERR(map_base) == -EBUSY);
+
+	if (IS_ERR(map_base))
+		return PTR_ERR(map_base);
+
+found:
+	info.type = ucs->type;
+	info.reg_lower = map_base + offset_in_page(ucs->phys_lower);
+	info.mask_lower = ucs->mmio.clksrc.mask;
+	info.bits_lower = ucs->bits_lower;
+	info.reg_upper = NULL;
+	if (ucs->phys_upper)
+		info.reg_upper = map_base + (size - PAGE_SIZE)
+			+ offset_in_page(ucs->phys_upper);
+	info.mask_upper = ucs->mask_upper;
+
+	return copy_to_user(u, &info, sizeof(*u));
+}
+
+static int mmio_ucs_open(struct inode *inode, struct file *file)
+{
+	struct clocksource_user_mmio *ucs;
+
+	if (file->f_mode & FMODE_WRITE)
+		return -EINVAL;
+
+	ucs = container_of(inode->i_cdev, typeof(*ucs), cdev);
+	file->private_data = ucs;
+
+	return 0;
+}
+
+static const struct file_operations mmio_ucs_fops = {
+	.owner		= THIS_MODULE,
+	.unlocked_ioctl = mmio_ucs_ioctl,
+	.open		= mmio_ucs_open,
+	.mmap		= mmio_ucs_mmap,
+};
+
+static int __init
+ucs_create_cdev(struct class *class, struct clocksource_user_mmio *ucs)
+{
+	int err;
+
+	ucs->dev = device_create(class, NULL,
+				MKDEV(MAJOR(user_mmio_devt), ucs->id),
+				ucs, "ucs/%d", ucs->id);
+	if (IS_ERR(ucs->dev))
+		return PTR_ERR(ucs->dev);
+
+	spin_lock_init(&ucs->lock);
+	hash_init(ucs->mappings);
+
+	cdev_init(&ucs->cdev, &mmio_ucs_fops);
+	ucs->cdev.kobj.parent = &ucs->dev->kobj;
+
+	err = cdev_add(&ucs->cdev, ucs->dev->devt, 1);
+	if (err < 0)
+		goto err_device_destroy;
+
+	return 0;
+
+err_device_destroy:
+	device_destroy(class, MKDEV(MAJOR(user_mmio_devt), ucs->id));
+	return err;
+}
+
+static unsigned long default_revmap(void *virt)
+{
+	struct vm_struct *vm;
+
+	vm = find_vm_area(virt);
+	if (!vm)
+		return 0;
+
+	return vm->phys_addr + (virt - vm->addr);
+}
+
+int __init clocksource_user_mmio_init(struct clocksource_user_mmio *ucs,
+				      const struct clocksource_mmio_regs *regs,
+				      unsigned long hz)
+{
+	static u64 (*user_types[CLKSRC_MMIO_TYPE_NR])(struct clocksource *) = {
+		[CLKSRC_MMIO_L_UP] = clocksource_mmio_readl_up,
+		[CLKSRC_MMIO_L_DOWN] = clocksource_mmio_readl_down,
+		[CLKSRC_DMMIO_L_UP] = clocksource_dual_mmio_readl_up,
+		[CLKSRC_MMIO_W_UP] = clocksource_mmio_readw_up,
+		[CLKSRC_MMIO_W_DOWN] = clocksource_mmio_readw_down,
+		[CLKSRC_DMMIO_W_UP] = clocksource_dual_mmio_readw_up,
+	};
+	const char *name = ucs->mmio.clksrc.name;
+	unsigned long phys_upper = 0, phys_lower;
+	enum clksrc_user_mmio_type type;
+	unsigned long (*revmap)(void *);
+	int err;
+
+	if (regs->bits_lower > 32 || regs->bits_lower < 16 ||
+	    regs->bits_upper > 32)
+		return -EINVAL;
+
+	for (type = 0; type < ARRAY_SIZE(user_types); type++)
+		if (ucs->mmio.clksrc.read == user_types[type])
+			break;
+
+	if (type == ARRAY_SIZE(user_types))
+		return -EINVAL;
+
+	if (!(ucs->mmio.clksrc.flags & CLOCK_SOURCE_IS_CONTINUOUS))
+		return -EINVAL;
+
+	revmap = regs->revmap;
+	if (!revmap)
+		revmap = default_revmap;
+
+	phys_lower = revmap(regs->reg_lower);
+	if (!phys_lower)
+		return -EINVAL;
+
+	if (regs->bits_upper) {
+		phys_upper = revmap(regs->reg_upper);
+		if (!phys_upper)
+			return -EINVAL;
+	}
+
+	ucs->mmio.reg = regs->reg_lower;
+	ucs->type = type;
+	ucs->bits_lower = regs->bits_lower;
+	ucs->reg_upper = regs->reg_upper;
+	ucs->mask_lower = CLOCKSOURCE_MASK(regs->bits_lower);
+	ucs->mask_upper = CLOCKSOURCE_MASK(regs->bits_upper);
+	ucs->phys_lower = phys_lower;
+	ucs->phys_upper = phys_upper;
+	spin_lock_init(&ucs->lock);
+
+	err = clocksource_register_hz(&ucs->mmio.clksrc, hz);
+	if (err < 0)
+		return err;
+
+	spin_lock(&user_clksrcs_lock);
+
+	ucs->id = user_clksrcs_count++;
+	if (ucs->id < CLKSRC_USER_MMIO_MAX)
+		list_add_tail(&ucs->link, &user_clksrcs);
+
+	spin_unlock(&user_clksrcs_lock);
+
+	if (ucs->id >= CLKSRC_USER_MMIO_MAX) {
+		pr_warn("%s: Too many clocksources\n", name);
+		err = -EAGAIN;
+		goto fail;
+	}
+
+	ucs->mmio.clksrc.vdso_type = CLOCKSOURCE_VDSO_MMIO + ucs->id;
+
+	if (user_mmio_class) {
+		err = ucs_create_cdev(user_mmio_class, ucs);
+		if (err < 0) {
+			pr_warn("%s: Failed to add character device\n", name);
+			goto fail;
+		}
+	}
+
+	return 0;
+
+fail:
+	clocksource_unregister(&ucs->mmio.clksrc);
+
+	return err;
+}
+
+int __init clocksource_user_single_mmio_init(
+	void __iomem *base, const char *name,
+	unsigned long hz, int rating, unsigned int bits,
+	u64 (*read)(struct clocksource *))
+{
+	struct clocksource_user_mmio *ucs;
+	struct clocksource_mmio_regs regs;
+	int ret;
+
+	ucs = kzalloc(sizeof(*ucs), GFP_KERNEL);
+	if (!ucs)
+		return -ENOMEM;
+
+	mmio_base_init(name, rating, bits, read, &ucs->mmio.clksrc);
+	regs.reg_lower = base;
+	regs.reg_upper = NULL;
+	regs.bits_lower = bits;
+	regs.bits_upper = 0;
+	regs.revmap = NULL;
+
+	ret = clocksource_user_mmio_init(ucs, &regs, hz);
+	if (ret)
+		kfree(ucs);
+
+	return ret;
+}
+
+static int __init mmio_clksrc_chr_dev_init(void)
+{
+	struct clocksource_user_mmio *ucs;
+	struct class *class;
+	int err;
+
+	class = class_create(THIS_MODULE, "mmio_ucs");
+	if (IS_ERR(class)) {
+		pr_err("couldn't create user mmio clocksources class\n");
+		return PTR_ERR(class);
+	}
+
+	err = alloc_chrdev_region(&user_mmio_devt, 0, CLKSRC_USER_MMIO_MAX,
+				  "mmio_ucs");
+	if (err < 0) {
+		pr_err("failed to allocate user mmio clocksources character devivces region\n");
+		goto err_class_destroy;
+	}
+
+	/*
+	 * Calling list_for_each_entry is safe here: clocksources are always
+	 * added to the list tail, never removed.
+	 */
+	spin_lock(&user_clksrcs_lock);
+	list_for_each_entry(ucs, &user_clksrcs, link) {
+		spin_unlock(&user_clksrcs_lock);
+
+		err = ucs_create_cdev(class, ucs);
+		if (err < 0)
+			pr_err("%s: Failed to add character device\n",
+			       ucs->mmio.clksrc.name);
+
+		spin_lock(&user_clksrcs_lock);
+	}
+	user_mmio_class = class;
+	spin_unlock(&user_clksrcs_lock);
+
+	return 0;
 
-	return clocksource_register_hz(&cs->clksrc, hz);
+err_class_destroy:
+	class_destroy(class);
+	return err;
 }
-EXPORT_SYMBOL_GPL(clocksource_mmio_init);
+device_initcall(mmio_clksrc_chr_dev_init);
diff --git a/kernel/drivers/clocksource/timer-imx-gpt.c b/kernel/drivers/clocksource/timer-imx-gpt.c
index 7b2c70f2f3..5c46458d72 100644
--- a/kernel/drivers/clocksource/timer-imx-gpt.c
+++ b/kernel/drivers/clocksource/timer-imx-gpt.c
@@ -163,8 +163,8 @@ static int __init mxc_clocksource_init(struct imx_timer *imxtm)
 	sched_clock_reg = reg;
 
 	sched_clock_register(mxc_read_sched_clock, 32, c);
-	return clocksource_mmio_init(reg, "mxc_timer1", c, 200, 32,
-			clocksource_mmio_readl_up);
+	return clocksource_user_single_mmio_init(reg, "mxc_timer1", c, 200, 32,
+					 clocksource_mmio_readl_up);
 }
 
 /* clock event */
@@ -264,7 +264,7 @@ static irqreturn_t mxc_timer_interrupt(int irq, void *dev_id)
 
 	imxtm->gpt->gpt_irq_acknowledge(imxtm);
 
-	ced->event_handler(ced);
+	clockevents_handle_event(ced);
 
 	return IRQ_HANDLED;
 }
@@ -274,7 +274,7 @@ static int __init mxc_clockevent_init(struct imx_timer *imxtm)
 	struct clock_event_device *ced = &imxtm->ced;
 
 	ced->name = "mxc_timer1";
-	ced->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_DYNIRQ;
+	ced->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_DYNIRQ | CLOCK_EVT_FEAT_PIPELINE;
 	ced->set_state_shutdown = mxc_shutdown;
 	ced->set_state_oneshot = mxc_set_oneshot;
 	ced->tick_resume = mxc_shutdown;
diff --git a/kernel/drivers/clocksource/timer-sun4i.c b/kernel/drivers/clocksource/timer-sun4i.c
index 0ba8155b82..43886c355c 100644
--- a/kernel/drivers/clocksource/timer-sun4i.c
+++ b/kernel/drivers/clocksource/timer-sun4i.c
@@ -19,6 +19,7 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqreturn.h>
+#include <linux/dovetail.h>
 #include <linux/sched_clock.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
@@ -135,7 +136,7 @@ static irqreturn_t sun4i_timer_interrupt(int irq, void *dev_id)
 	struct timer_of *to = to_timer_of(evt);
 
 	sun4i_timer_clear_interrupt(timer_of_base(to));
-	evt->event_handler(evt);
+	clockevents_handle_event(evt);
 
 	return IRQ_HANDLED;
 }
@@ -146,7 +147,7 @@ static struct timer_of to = {
 	.clkevt = {
 		.name = "sun4i_tick",
 		.rating = 350,
-		.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT,
+		.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PIPELINE,
 		.set_state_shutdown = sun4i_clkevt_shutdown,
 		.set_state_periodic = sun4i_clkevt_set_periodic,
 		.set_state_oneshot = sun4i_clkevt_set_oneshot,
diff --git a/kernel/drivers/clocksource/timer-ti-dm-systimer.c b/kernel/drivers/clocksource/timer-ti-dm-systimer.c
index 2737407ff0..345569d678 100644
--- a/kernel/drivers/clocksource/timer-ti-dm-systimer.c
+++ b/kernel/drivers/clocksource/timer-ti-dm-systimer.c
@@ -57,7 +57,7 @@ struct dmtimer_clockevent {
 };
 
 struct dmtimer_clocksource {
-	struct clocksource dev;
+	struct clocksource_user_mmio mmio;
 	struct dmtimer_systimer t;
 	unsigned int loadval;
 };
@@ -437,7 +437,7 @@ static irqreturn_t dmtimer_clockevent_interrupt(int irq, void *data)
 	struct dmtimer_systimer *t = &clkevt->t;
 
 	writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_stat);
-	clkevt->dev.event_handler(&clkevt->dev);
+	clockevents_handle_event(&clkevt->dev);
 
 	return IRQ_HANDLED;
 }
@@ -548,7 +548,7 @@ static int __init dmtimer_clkevt_init_common(struct dmtimer_clockevent *clkevt,
 	 * We mostly use cpuidle_coupled with ARM local timers for runtime,
 	 * so there's probably no use for CLOCK_EVT_FEAT_DYNIRQ here.
 	 */
-	dev->features = features;
+	dev->features = features | CLOCK_EVT_FEAT_PIPELINE;
 	dev->rating = rating;
 	dev->set_next_event = dmtimer_set_next_event;
 	dev->set_state_shutdown = dmtimer_clockevent_shutdown;
@@ -706,15 +706,7 @@ static int __init dmtimer_percpu_quirk_init(struct device_node *np, u32 pa)
 static struct dmtimer_clocksource *
 to_dmtimer_clocksource(struct clocksource *cs)
 {
-	return container_of(cs, struct dmtimer_clocksource, dev);
-}
-
-static u64 dmtimer_clocksource_read_cycles(struct clocksource *cs)
-{
-	struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs);
-	struct dmtimer_systimer *t = &clksrc->t;
-
-	return (u64)readl_relaxed(t->base + t->counter);
+	return container_of(cs, struct dmtimer_clocksource, mmio.mmio.clksrc);
 }
 
 static void __iomem *dmtimer_sched_clock_counter;
@@ -753,6 +745,7 @@ static void dmtimer_clocksource_resume(struct clocksource *cs)
 static int __init dmtimer_clocksource_init(struct device_node *np)
 {
 	struct dmtimer_clocksource *clksrc;
+	struct clocksource_mmio_regs mmr;
 	struct dmtimer_systimer *t;
 	struct clocksource *dev;
 	int error;
@@ -761,7 +754,7 @@ static int __init dmtimer_clocksource_init(struct device_node *np)
 	if (!clksrc)
 		return -ENOMEM;
 
-	dev = &clksrc->dev;
+	dev = &clksrc->mmio.mmio.clksrc;
 	t = &clksrc->t;
 
 	error = dmtimer_systimer_setup(np, t);
@@ -770,7 +763,7 @@ static int __init dmtimer_clocksource_init(struct device_node *np)
 
 	dev->name = "dmtimer";
 	dev->rating = 300;
-	dev->read = dmtimer_clocksource_read_cycles;
+	dev->read = clocksource_mmio_readl_up,
 	dev->mask = CLOCKSOURCE_MASK(32);
 	dev->flags = CLOCK_SOURCE_IS_CONTINUOUS;
 
@@ -793,7 +786,13 @@ static int __init dmtimer_clocksource_init(struct device_node *np)
 		sched_clock_register(dmtimer_read_sched_clock, 32, t->rate);
 	}
 
-	if (clocksource_register_hz(dev, t->rate))
+	mmr.reg_lower = t->base + t->counter;
+	mmr.bits_lower = 32;
+	mmr.reg_upper = 0;
+	mmr.bits_upper = 0;
+	mmr.revmap = NULL;
+
+	if (clocksource_user_mmio_init(&clksrc->mmio, &mmr, t->rate))
 		pr_err("Could not register clocksource %pOF\n", np);
 
 	return 0;
diff --git a/kernel/drivers/cpuidle/cpuidle.c b/kernel/drivers/cpuidle/cpuidle.c
index ab77a363a9..9c6004dce7 100644
--- a/kernel/drivers/cpuidle/cpuidle.c
+++ b/kernel/drivers/cpuidle/cpuidle.c
@@ -17,6 +17,7 @@
 #include <linux/pm_qos.h>
 #include <linux/cpu.h>
 #include <linux/cpuidle.h>
+#include <linux/irq_pipeline.h>
 #include <linux/ktime.h>
 #include <linux/hrtimer.h>
 #include <linux/module.h>
@@ -218,6 +219,22 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
 	target_state = &drv->states[index];
 	broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
 
+	/*
+	 * A companion core running on the oob stage of the IRQ
+	 * pipeline may deny switching to a deeper C-state. If so,
+	 * call the default idle routine instead. If the core cannot
+	 * bear with the latency induced by the default idling
+	 * operation, then CPUIDLE is not usable and should be
+	 * disabled at build time. The in-band stage is currently
+	 * stalled, hard irqs are on. irq_cpuidle_enter() leaves us
+	 * stalled but returns with hard irqs off so that no event may
+	 * sneak in until we actually go idle.
+	 */
+	if (!irq_cpuidle_enter(dev, target_state)) {
+		default_idle_call();
+		return -EBUSY;
+	}
+
 	/*
 	 * Tell the time framework to switch to a broadcast timer because our
 	 * local timer will be shut down.  If a local timer is used from another
@@ -247,6 +264,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
 		rcu_idle_enter();
 	entered_state = target_state->enter(dev, drv, index);
+	hard_cond_local_irq_enable();
 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
 		rcu_idle_exit();
 	start_critical_timings();
diff --git a/kernel/drivers/cpuidle/poll_state.c b/kernel/drivers/cpuidle/poll_state.c
index f7e83613ae..1245138d70 100644
--- a/kernel/drivers/cpuidle/poll_state.c
+++ b/kernel/drivers/cpuidle/poll_state.c
@@ -17,7 +17,7 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
 
 	dev->poll_time_limit = false;
 
-	local_irq_enable();
+	local_irq_enable_full();
 	if (!current_set_polling_and_test()) {
 		unsigned int loop_count = 0;
 		u64 limit;
diff --git a/kernel/drivers/dma/Kconfig b/kernel/drivers/dma/Kconfig
index 08013345d1..821fe3d3cc 100644
--- a/kernel/drivers/dma/Kconfig
+++ b/kernel/drivers/dma/Kconfig
@@ -47,6 +47,10 @@ config DMA_ENGINE
 config DMA_VIRTUAL_CHANNELS
 	tristate
 
+config DMA_VIRTUAL_CHANNELS_OOB
+	def_bool n
+	depends on DMA_VIRTUAL_CHANNELS && DOVETAIL
+
 config DMA_ACPI
 	def_bool y
 	depends on ACPI
@@ -138,6 +142,13 @@ config DMA_BCM2835
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
 
+config DMA_BCM2835_OOB
+	bool "Out-of-band support for BCM2835 DMA"
+	depends on DMA_BCM2835 && DOVETAIL
+	select DMA_VIRTUAL_CHANNELS_OOB
+	help
+	  Enable out-of-band requests to BCM2835 DMA.
+
 config DMA_JZ4780
 	tristate "JZ4780 DMA support"
 	depends on MIPS || COMPILE_TEST
@@ -275,6 +286,13 @@ config IMX_SDMA
 	  Support the i.MX SDMA engine. This engine is integrated into
 	  Freescale i.MX25/31/35/51/53/6 chips.
 
+config IMX_SDMA_OOB
+	bool "Out-of-band support for i.MX SDMA"
+	depends on IMX_SDMA && DOVETAIL
+	select DMA_VIRTUAL_CHANNELS_OOB
+	help
+	  Enable out-of-band requests to i.MX SDMA.
+
 config INTEL_IDMA64
 	tristate "Intel integrated DMA 64-bit support"
 	select DMA_ENGINE
diff --git a/kernel/drivers/dma/bcm2835-dma.c b/kernel/drivers/dma/bcm2835-dma.c
index 630dfbb01a..6161f76440 100644
--- a/kernel/drivers/dma/bcm2835-dma.c
+++ b/kernel/drivers/dma/bcm2835-dma.c
@@ -29,6 +29,7 @@
 #include <linux/slab.h>
 #include <linux/io.h>
 #include <linux/spinlock.h>
+#include <linux/irqstage.h>
 #include <linux/of.h>
 #include <linux/of_dma.h>
 
@@ -435,10 +436,20 @@ static void bcm2835_dma_abort(struct bcm2835_chan *c)
 	writel(BCM2835_DMA_RESET, chan_base + BCM2835_DMA_CS);
 }
 
+static inline void bcm2835_dma_enable_channel(struct bcm2835_chan *c)
+{
+	writel(c->desc->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);
+	writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS);
+}
+
+static inline bool bcm2835_dma_oob_capable(void)
+{
+	return IS_ENABLED(CONFIG_DMA_BCM2835_OOB);
+}
+
 static void bcm2835_dma_start_desc(struct bcm2835_chan *c)
 {
 	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
-	struct bcm2835_desc *d;
 
 	if (!vd) {
 		c->desc = NULL;
@@ -447,10 +458,41 @@ static void bcm2835_dma_start_desc(struct bcm2835_chan *c)
 
 	list_del(&vd->node);
 
-	c->desc = d = to_bcm2835_dma_desc(&vd->tx);
+	c->desc = to_bcm2835_dma_desc(&vd->tx);
+	if (!bcm2835_dma_oob_capable() || !vchan_oob_pulsed(vd))
+		bcm2835_dma_enable_channel(c);
+}
 
-	writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);
-	writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS);
+static bool do_channel(struct bcm2835_chan *c, struct bcm2835_desc *d)
+{
+	struct dmaengine_desc_callback cb;
+
+	if (running_oob()) {
+		if (!vchan_oob_handled(&d->vd))
+			return false;
+		dmaengine_desc_get_callback(&d->vd.tx, &cb);
+		if (dmaengine_desc_callback_valid(&cb)) {
+			vchan_unlock(&c->vc);
+			dmaengine_desc_callback_invoke(&cb, NULL);
+			vchan_lock(&c->vc);
+		}
+		return true;
+	}
+
+	if (d->cyclic) {
+		/* call the cyclic callback */
+		vchan_cyclic_callback(&d->vd);
+	} else if (!readl(c->chan_base + BCM2835_DMA_ADDR)) {
+		vchan_cookie_complete(&c->desc->vd);
+		bcm2835_dma_start_desc(c);
+	}
+
+	return true;
+}
+
+static inline bool is_base_irq_handler(void)
+{
+	return !bcm2835_dma_oob_capable() || running_oob();
 }
 
 static irqreturn_t bcm2835_dma_callback(int irq, void *data)
@@ -460,7 +502,7 @@ static irqreturn_t bcm2835_dma_callback(int irq, void *data)
 	unsigned long flags;
 
 	/* check the shared interrupt */
-	if (c->irq_flags & IRQF_SHARED) {
+	if (is_base_irq_handler() && c->irq_flags & IRQF_SHARED) {
 		/* check if the interrupt is enabled */
 		flags = readl(c->chan_base + BCM2835_DMA_CS);
 		/* if not set then we are not the reason for the irq */
@@ -468,7 +510,8 @@ static irqreturn_t bcm2835_dma_callback(int irq, void *data)
 			return IRQ_NONE;
 	}
 
-	spin_lock_irqsave(&c->vc.lock, flags);
+	/* CAUTION: If running in-band, hard irqs are on. */
+	vchan_lock_irqsave(&c->vc, flags);
 
 	/*
 	 * Clear the INT flag to receive further interrupts. Keep the channel
@@ -477,22 +520,27 @@ static irqreturn_t bcm2835_dma_callback(int irq, void *data)
 	 * if this IRQ handler is threaded.) If the channel is finished, it
 	 * will remain idle despite the ACTIVE flag being set.
 	 */
-	writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE,
-	       c->chan_base + BCM2835_DMA_CS);
+	if (is_base_irq_handler())
+		writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE,
+			c->chan_base + BCM2835_DMA_CS);
 
 	d = c->desc;
+	if (!d)
+		goto out;
 
-	if (d) {
-		if (d->cyclic) {
-			/* call the cyclic callback */
-			vchan_cyclic_callback(&d->vd);
-		} else if (!readl(c->chan_base + BCM2835_DMA_ADDR)) {
-			vchan_cookie_complete(&c->desc->vd);
-			bcm2835_dma_start_desc(c);
-		}
+	if (bcm2835_dma_oob_capable() && running_oob()) {
+		/*
+		 * If we cannot process this from the out-of-band
+		 * stage, schedule a callback from in-band context.
+		 */
+		if (!do_channel(c, d))
+			irq_post_inband(irq);
+	} else {
+		do_channel(c, d);
 	}
 
-	spin_unlock_irqrestore(&c->vc.lock, flags);
+out:
+	vchan_unlock_irqrestore(&c->vc, flags);
 
 	return IRQ_HANDLED;
 }
@@ -571,7 +619,7 @@ static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan,
 	if (ret == DMA_COMPLETE || !txstate)
 		return ret;
 
-	spin_lock_irqsave(&c->vc.lock, flags);
+	vchan_lock_irqsave(&c->vc, flags);
 	vd = vchan_find_desc(&c->vc, cookie);
 	if (vd) {
 		txstate->residue =
@@ -592,7 +640,7 @@ static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan,
 		txstate->residue = 0;
 	}
 
-	spin_unlock_irqrestore(&c->vc.lock, flags);
+	vchan_unlock_irqrestore(&c->vc, flags);
 
 	return ret;
 }
@@ -602,13 +650,36 @@ static void bcm2835_dma_issue_pending(struct dma_chan *chan)
 	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
 	unsigned long flags;
 
-	spin_lock_irqsave(&c->vc.lock, flags);
+	vchan_lock_irqsave(&c->vc, flags);
 	if (vchan_issue_pending(&c->vc) && !c->desc)
 		bcm2835_dma_start_desc(c);
 
-	spin_unlock_irqrestore(&c->vc.lock, flags);
+	vchan_unlock_irqrestore(&c->vc, flags);
 }
 
+#ifdef CONFIG_DMA_BCM2835_OOB
+static int bcm2835_dma_pulse_oob(struct dma_chan *chan)
+{
+	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
+	unsigned long flags;
+	int ret = -EIO;
+
+	vchan_lock_irqsave(&c->vc, flags);
+	if (c->desc && vchan_oob_pulsed(&c->desc->vd)) {
+		bcm2835_dma_enable_channel(c);
+		ret = 0;
+	}
+	vchan_unlock_irqrestore(&c->vc, flags);
+
+	return ret;
+}
+#else
+static int bcm2835_dma_pulse_oob(struct dma_chan *chan)
+{
+	return -ENOTSUPP;
+}
+#endif
+
 static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy(
 	struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
 	size_t len, unsigned long flags)
@@ -650,6 +721,15 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg(
 	u32 extra = BCM2835_DMA_INT_EN;
 	size_t frames;
 
+	if (!bcm2835_dma_oob_capable()) {
+		if (flags & (DMA_OOB_INTERRUPT|DMA_OOB_PULSE)) {
+			dev_err(chan->device->dev,
+				"%s: out-of-band slave transfers disabled\n",
+				__func__);
+			return NULL;
+		}
+	}
+
 	if (!is_slave_direction(direction)) {
 		dev_err(chan->device->dev,
 			"%s: bad direction?\n", __func__);
@@ -715,7 +795,21 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(
 		return NULL;
 	}
 
-	if (flags & DMA_PREP_INTERRUPT)
+	if (!bcm2835_dma_oob_capable()) {
+		if (flags & DMA_OOB_INTERRUPT) {
+			dev_err(chan->device->dev,
+				"%s: out-of-band cyclic transfers disabled\n",
+				__func__);
+			return NULL;
+		}
+	} else if (flags & DMA_OOB_PULSE) {
+		dev_err(chan->device->dev,
+			"%s: no pulse mode with out-of-band cyclic transfers\n",
+			__func__);
+		return NULL;
+	}
+
+	if (flags & (DMA_PREP_INTERRUPT|DMA_OOB_INTERRUPT))
 		extra |= BCM2835_DMA_INT_EN;
 	else
 		period_len = buf_len;
@@ -791,7 +885,7 @@ static int bcm2835_dma_terminate_all(struct dma_chan *chan)
 	unsigned long flags;
 	LIST_HEAD(head);
 
-	spin_lock_irqsave(&c->vc.lock, flags);
+	vchan_lock_irqsave(&c->vc, flags);
 
 	/* stop DMA activity */
 	if (c->desc) {
@@ -801,7 +895,7 @@ static int bcm2835_dma_terminate_all(struct dma_chan *chan)
 	}
 
 	vchan_get_all_descriptors(&c->vc, &head);
-	spin_unlock_irqrestore(&c->vc.lock, flags);
+	vchan_unlock_irqrestore(&c->vc, flags);
 	vchan_dma_desc_free_list(&c->vc, &head);
 
 	return 0;
@@ -912,11 +1006,13 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
 	dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
 	dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask);
 	dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
+	dma_cap_set(DMA_OOB, od->ddev.cap_mask);
 	dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
 	od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources;
 	od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources;
 	od->ddev.device_tx_status = bcm2835_dma_tx_status;
 	od->ddev.device_issue_pending = bcm2835_dma_issue_pending;
+	od->ddev.device_pulse_oob = bcm2835_dma_pulse_oob;
 	od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic;
 	od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg;
 	od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy;
@@ -982,10 +1078,10 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
 			continue;
 
 		/* check if there are other channels that also use this irq */
-		irq_flags = 0;
+		irq_flags = IS_ENABLED(CONFIG_DMA_BCM2835_OOB) ? IRQF_OOB : 0;
 		for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++)
 			if ((i != j) && (irq[j] == irq[i])) {
-				irq_flags = IRQF_SHARED;
+				irq_flags |= IRQF_SHARED;
 				break;
 			}
 
diff --git a/kernel/drivers/dma/dmaengine.c b/kernel/drivers/dma/dmaengine.c
index af3ee288bc..e79a94d60c 100644
--- a/kernel/drivers/dma/dmaengine.c
+++ b/kernel/drivers/dma/dmaengine.c
@@ -578,7 +578,8 @@ int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
 
 	/* check if the channel supports slave transactions */
 	if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||
-	      test_bit(DMA_CYCLIC, device->cap_mask.bits)))
+	      test_bit(DMA_CYCLIC, device->cap_mask.bits) ||
+	      test_bit(DMA_OOB, device->cap_mask.bits)))
 		return -ENXIO;
 
 	/*
@@ -1209,6 +1210,13 @@ int dma_async_device_register(struct dma_device *device)
 		return -EIO;
 	}
 
+	if (dma_has_cap(DMA_OOB, device->cap_mask) && !device->device_pulse_oob) {
+		dev_err(device->dev,
+			"Device claims capability %s, but pulse handler is not defined\n",
+			"DMA_OOB");
+		return -EIO;
+	}
+
 	if (dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && !device->device_prep_interleaved_dma) {
 		dev_err(device->dev,
 			"Device claims capability %s, but op is not defined\n",
diff --git a/kernel/drivers/dma/imx-sdma.c b/kernel/drivers/dma/imx-sdma.c
index 2283dcd8bf..3648d3c62f 100644
--- a/kernel/drivers/dma/imx-sdma.c
+++ b/kernel/drivers/dma/imx-sdma.c
@@ -444,6 +444,10 @@ struct sdma_engine {
 	struct sdma_buffer_descriptor	*bd0;
 	/* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/
 	bool				clk_ratio;
+#ifdef CONFIG_IMX_SDMA_OOB
+	hard_spinlock_t			oob_lock;
+	u32				pending_stat;
+#endif
 };
 
 static int sdma_config_write(struct dma_chan *chan,
@@ -748,6 +752,11 @@ static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
 	return container_of(t, struct sdma_desc, vd.tx);
 }
 
+static inline bool sdma_oob_capable(void)
+{
+	return IS_ENABLED(CONFIG_IMX_SDMA_OOB);
+}
+
 static void sdma_start_desc(struct sdma_channel *sdmac)
 {
 	struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
@@ -765,7 +774,8 @@ static void sdma_start_desc(struct sdma_channel *sdmac)
 
 	sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
 	sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
-	sdma_enable_channel(sdma, sdmac->channel);
+	if (!sdma_oob_capable() || !vchan_oob_pulsed(vd))
+		sdma_enable_channel(sdma, sdmac->channel);
 }
 
 static void sdma_update_channel_loop(struct sdma_channel *sdmac)
@@ -809,9 +819,9 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac)
 		 * SDMA transaction status by the time the client tasklet is
 		 * executed.
 		 */
-		spin_unlock(&sdmac->vc.lock);
+		vchan_unlock(&sdmac->vc);
 		dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
-		spin_lock(&sdmac->vc.lock);
+		vchan_lock(&sdmac->vc);
 
 		if (error)
 			sdmac->status = old_status;
@@ -821,20 +831,21 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac)
 static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
 {
 	struct sdma_channel *sdmac = (struct sdma_channel *) data;
+	struct sdma_desc *desc = sdmac->desc;
 	struct sdma_buffer_descriptor *bd;
 	int i, error = 0;
 
-	sdmac->desc->chn_real_count = 0;
+	desc->chn_real_count = 0;
 	/*
 	 * non loop mode. Iterate over all descriptors, collect
 	 * errors and call callback function
 	 */
-	for (i = 0; i < sdmac->desc->num_bd; i++) {
-		bd = &sdmac->desc->bd[i];
+	for (i = 0; i < desc->num_bd; i++) {
+		bd = &desc->bd[i];
 
 		 if (bd->mode.status & (BD_DONE | BD_RROR))
 			error = -EIO;
-		 sdmac->desc->chn_real_count += bd->mode.count;
+		 desc->chn_real_count += bd->mode.count;
 	}
 
 	if (error)
@@ -843,37 +854,84 @@ static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
 		sdmac->status = DMA_COMPLETE;
 }
 
-static irqreturn_t sdma_int_handler(int irq, void *dev_id)
+static unsigned long sdma_do_channels(struct sdma_engine *sdma,
+				unsigned long stat)
 {
-	struct sdma_engine *sdma = dev_id;
-	unsigned long stat;
+	unsigned long mask = stat;
 
-	stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
-	writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
-	/* channel 0 is special and not handled here, see run_channel0() */
-	stat &= ~1;
-
-	while (stat) {
-		int channel = fls(stat) - 1;
+	while (mask) {
+		int channel = fls(mask) - 1;
 		struct sdma_channel *sdmac = &sdma->channel[channel];
 		struct sdma_desc *desc;
 
-		spin_lock(&sdmac->vc.lock);
+		vchan_lock(&sdmac->vc);
 		desc = sdmac->desc;
 		if (desc) {
+			if (running_oob() && !vchan_oob_handled(&desc->vd))
+				goto next;
 			if (sdmac->flags & IMX_DMA_SG_LOOP) {
 				sdma_update_channel_loop(sdmac);
 			} else {
 				mxc_sdma_handle_channel_normal(sdmac);
+				if (running_oob()) {
+					vchan_unlock(&sdmac->vc);
+					dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
+					__clear_bit(channel, &stat);
+					goto next_unlocked;
+				}
 				vchan_cookie_complete(&desc->vd);
 				sdma_start_desc(sdmac);
 			}
 		}
-
-		spin_unlock(&sdmac->vc.lock);
 		__clear_bit(channel, &stat);
+	next:
+		vchan_unlock(&sdmac->vc);
+	next_unlocked:
+		__clear_bit(channel, &mask);
 	}
 
+	return stat;
+}
+
+static irqreturn_t sdma_int_handler(int irq, void *dev_id)
+{
+	struct sdma_engine *sdma = dev_id;
+	unsigned long stat, flags __maybe_unused;
+
+#ifdef CONFIG_IMX_SDMA_OOB
+	if (running_oob()) {
+		stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
+		writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
+		/*
+		 * Locking is only to guard against IRQ migration with
+		 * a delayed in-band event running from a remote CPU
+		 * after some IRQ routing changed the affinity of the
+		 * out-of-band handler in the meantime.
+		 */
+		stat = sdma_do_channels(sdma, stat & ~1);
+		if (stat) {
+			raw_spin_lock(&sdma->oob_lock);
+			sdma->pending_stat |= stat;
+			raw_spin_unlock(&sdma->oob_lock);
+			/* Call us back from in-band context. */
+			irq_post_inband(irq);
+		}
+		return IRQ_HANDLED;
+	}
+
+	/* In-band IRQ context: stalled, but hard irqs are on. */
+	raw_spin_lock_irqsave(&sdma->oob_lock, flags);
+	stat = sdma->pending_stat;
+	sdma->pending_stat = 0;
+	raw_spin_unlock_irqrestore(&sdma->oob_lock, flags);
+	sdma_do_channels(sdma, stat);
+#else
+	stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
+	writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
+	/* channel 0 is special and not handled here, see run_channel0() */
+	sdma_do_channels(sdma, stat & ~1);
+#endif
+
 	return IRQ_HANDLED;
 }
 
@@ -1060,9 +1118,9 @@ static void sdma_channel_terminate_work(struct work_struct *work)
 	 */
 	usleep_range(1000, 2000);
 
-	spin_lock_irqsave(&sdmac->vc.lock, flags);
+	vchan_lock_irqsave(&sdmac->vc, flags);
 	vchan_get_all_descriptors(&sdmac->vc, &head);
-	spin_unlock_irqrestore(&sdmac->vc.lock, flags);
+	vchan_unlock_irqrestore(&sdmac->vc, flags);
 	vchan_dma_desc_free_list(&sdmac->vc, &head);
 }
 
@@ -1071,18 +1129,19 @@ static int sdma_terminate_all(struct dma_chan *chan)
 	struct sdma_channel *sdmac = to_sdma_chan(chan);
 	unsigned long flags;
 
-	spin_lock_irqsave(&sdmac->vc.lock, flags);
+	vchan_lock_irqsave(&sdmac->vc, flags);
 
 	sdma_disable_channel(chan);
 
 	if (sdmac->desc) {
 		vchan_terminate_vdesc(&sdmac->desc->vd);
 		sdmac->desc = NULL;
+		vchan_unlock_irqrestore(&sdmac->vc, flags);
 		schedule_work(&sdmac->terminate_worker);
+	} else {
+		vchan_unlock_irqrestore(&sdmac->vc, flags);
 	}
 
-	spin_unlock_irqrestore(&sdmac->vc.lock, flags);
-
 	return 0;
 }
 
@@ -1441,6 +1500,15 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
 	struct scatterlist *sg;
 	struct sdma_desc *desc;
 
+	if (!sdma_oob_capable()) {
+		if (flags & (DMA_OOB_INTERRUPT|DMA_OOB_PULSE)) {
+			dev_err(sdma->dev,
+				"%s: out-of-band slave transfers disabled\n",
+				__func__);
+			return NULL;
+		}
+	}
+
 	sdma_config_write(chan, &sdmac->slave_config, direction);
 
 	desc = sdma_transfer_init(sdmac, direction, sg_len);
@@ -1492,7 +1560,8 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
 
 		if (i + 1 == sg_len) {
 			param |= BD_INTR;
-			param |= BD_LAST;
+			if (!sdma_oob_capable() || !(flags & DMA_OOB_PULSE))
+				param |= BD_LAST;
 			param &= ~BD_CONT;
 		}
 
@@ -1527,6 +1596,20 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
 
 	dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
 
+	if (!sdma_oob_capable()) {
+		if (flags & (DMA_OOB_INTERRUPT|DMA_OOB_PULSE)) {
+			dev_err(sdma->dev,
+				"%s: out-of-band cyclic transfers disabled\n",
+				__func__);
+			return NULL;
+		}
+	} else if (flags & DMA_OOB_PULSE) {
+		dev_err(chan->device->dev,
+			"%s: no pulse mode with out-of-band cyclic transfers\n",
+			__func__);
+		return NULL;
+	}
+
 	sdma_config_write(chan, &sdmac->slave_config, direction);
 
 	desc = sdma_transfer_init(sdmac, direction, num_periods);
@@ -1649,7 +1732,7 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan,
 	if (ret == DMA_COMPLETE || !txstate)
 		return ret;
 
-	spin_lock_irqsave(&sdmac->vc.lock, flags);
+	vchan_lock_irqsave(&sdmac->vc, flags);
 
 	vd = vchan_find_desc(&sdmac->vc, cookie);
 	if (vd)
@@ -1667,7 +1750,7 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan,
 		residue = 0;
 	}
 
-	spin_unlock_irqrestore(&sdmac->vc.lock, flags);
+	vchan_unlock_irqrestore(&sdmac->vc, flags);
 
 	dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
 			 residue);
@@ -1680,11 +1763,38 @@ static void sdma_issue_pending(struct dma_chan *chan)
 	struct sdma_channel *sdmac = to_sdma_chan(chan);
 	unsigned long flags;
 
-	spin_lock_irqsave(&sdmac->vc.lock, flags);
+	vchan_lock_irqsave(&sdmac->vc, flags);
 	if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
 		sdma_start_desc(sdmac);
-	spin_unlock_irqrestore(&sdmac->vc.lock, flags);
+	vchan_unlock_irqrestore(&sdmac->vc, flags);
+}
+
+#ifdef CONFIG_IMX_SDMA_OOB
+static int sdma_pulse_oob(struct dma_chan *chan)
+{
+	struct sdma_channel *sdmac = to_sdma_chan(chan);
+	struct sdma_desc *desc = sdmac->desc;
+	unsigned long flags;
+	int n, ret = -EIO;
+
+	vchan_lock_irqsave(&sdmac->vc, flags);
+	if (desc && vchan_oob_pulsed(&desc->vd)) {
+		for (n = 0; n < desc->num_bd - 1; n++)
+			desc->bd[n].mode.status |= BD_DONE;
+		desc->bd[n].mode.status |= BD_DONE|BD_WRAP;
+		sdma_enable_channel(sdmac->sdma, sdmac->channel);
+		ret = 0;
+	}
+	vchan_unlock_irqrestore(&sdmac->vc, flags);
+
+	return ret;
 }
+#else
+static int sdma_pulse_oob(struct dma_chan *chan)
+{
+	return -ENOTSUPP;
+}
+#endif
 
 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1	34
 #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2	38
@@ -1920,6 +2030,9 @@ static int sdma_init(struct sdma_engine *sdma)
 	clk_disable(sdma->clk_ipg);
 	clk_disable(sdma->clk_ahb);
 
+#ifdef CONFIG_IMX_SDMA_OOB
+	raw_spin_lock_init(&sdma->oob_lock);
+#endif
 	return 0;
 
 err_dma_alloc:
@@ -2035,8 +2148,9 @@ static int sdma_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_clk;
 
-	ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma",
-			       sdma);
+	ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler,
+			IS_ENABLED(CONFIG_IMX_SDMA_OOB) ? IRQF_OOB : 0,
+			"sdma", sdma);
 	if (ret)
 		goto err_irq;
 
@@ -2055,6 +2169,7 @@ static int sdma_probe(struct platform_device *pdev)
 
 	dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
 	dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
+	dma_cap_set(DMA_OOB, sdma->dma_device.cap_mask);
 	dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask);
 
 	INIT_LIST_HEAD(&sdma->dma_device.channels);
@@ -2106,6 +2221,7 @@ static int sdma_probe(struct platform_device *pdev)
 	sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
 	sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy;
 	sdma->dma_device.device_issue_pending = sdma_issue_pending;
+	sdma->dma_device.device_pulse_oob = sdma_pulse_oob;
 	sdma->dma_device.copy_align = 2;
 	dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT);
 
@@ -2160,6 +2276,16 @@ static int sdma_probe(struct platform_device *pdev)
 		}
 	}
 
+	/*
+	 * Keep the clocks enabled at any time if we plan to use the
+	 * DMA from out-of-band context, bumping their refcount to
+	 * keep them on until sdma_remove() is called eventually.
+	 */
+	if (IS_ENABLED(CONFIG_IMX_SDMA_OOB)) {
+		clk_enable(sdma->clk_ipg);
+		clk_enable(sdma->clk_ahb);
+	}
+
 	return 0;
 
 err_register:
@@ -2178,6 +2304,11 @@ static int sdma_remove(struct platform_device *pdev)
 	struct sdma_engine *sdma = platform_get_drvdata(pdev);
 	int i;
 
+	if (IS_ENABLED(CONFIG_IMX_SDMA_OOB)) {
+		clk_disable(sdma->clk_ahb);
+		clk_disable(sdma->clk_ipg);
+	}
+
 	devm_free_irq(&pdev->dev, sdma->irq, sdma);
 	dma_async_device_unregister(&sdma->dma_device);
 	kfree(sdma->script_addrs);
diff --git a/kernel/drivers/dma/virt-dma.c b/kernel/drivers/dma/virt-dma.c
index a6f4265be0..89e011699a 100644
--- a/kernel/drivers/dma/virt-dma.c
+++ b/kernel/drivers/dma/virt-dma.c
@@ -23,11 +23,11 @@ dma_cookie_t vchan_tx_submit(struct dma_async_tx_descriptor *tx)
 	unsigned long flags;
 	dma_cookie_t cookie;
 
-	spin_lock_irqsave(&vc->lock, flags);
+	vchan_lock_irqsave(vc, flags);
 	cookie = dma_cookie_assign(tx);
 
 	list_move_tail(&vd->node, &vc->desc_submitted);
-	spin_unlock_irqrestore(&vc->lock, flags);
+	vchan_unlock_irqrestore(vc, flags);
 
 	dev_dbg(vc->chan.device->dev, "vchan %p: txd %p[%x]: submitted\n",
 		vc, vd, cookie);
@@ -52,9 +52,9 @@ int vchan_tx_desc_free(struct dma_async_tx_descriptor *tx)
 	struct virt_dma_desc *vd = to_virt_desc(tx);
 	unsigned long flags;
 
-	spin_lock_irqsave(&vc->lock, flags);
+	vchan_lock_irqsave(vc, flags);
 	list_del(&vd->node);
-	spin_unlock_irqrestore(&vc->lock, flags);
+	vchan_unlock_irqrestore(vc, flags);
 
 	dev_dbg(vc->chan.device->dev, "vchan %p: txd %p[%x]: freeing\n",
 		vc, vd, vd->tx.cookie);
@@ -87,7 +87,7 @@ static void vchan_complete(struct tasklet_struct *t)
 	struct dmaengine_desc_callback cb;
 	LIST_HEAD(head);
 
-	spin_lock_irq(&vc->lock);
+	vchan_lock_irq(vc);
 	list_splice_tail_init(&vc->desc_completed, &head);
 	vd = vc->cyclic;
 	if (vd) {
@@ -96,7 +96,7 @@ static void vchan_complete(struct tasklet_struct *t)
 	} else {
 		memset(&cb, 0, sizeof(cb));
 	}
-	spin_unlock_irq(&vc->lock);
+	vchan_unlock_irq(vc);
 
 	dmaengine_desc_callback_invoke(&cb, &vd->tx_result);
 
@@ -120,11 +120,119 @@ void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head)
 }
 EXPORT_SYMBOL_GPL(vchan_dma_desc_free_list);
 
+#ifdef CONFIG_DMA_VIRTUAL_CHANNELS_OOB
+
+static void inband_init_chan_lock(struct virt_dma_chan *vc)
+{
+	spin_lock_init(&vc->lock);
+}
+
+static void inband_lock_chan(struct virt_dma_chan *vc)
+{
+	spin_lock(&vc->lock);
+}
+
+static void inband_unlock_chan(struct virt_dma_chan *vc)
+{
+	spin_unlock(&vc->lock);
+}
+
+static void inband_lock_irq_chan(struct virt_dma_chan *vc)
+{
+	spin_lock_irq(&vc->lock);
+}
+
+static void inband_unlock_irq_chan(struct virt_dma_chan *vc)
+{
+	spin_unlock_irq(&vc->lock);
+}
+
+static unsigned long inband_lock_irqsave_chan(struct virt_dma_chan *vc)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&vc->lock, flags);
+
+	return flags;
+}
+
+static void inband_unlock_irqrestore_chan(struct virt_dma_chan *vc,
+			unsigned long flags)
+{
+	spin_unlock_irqrestore(&vc->lock, flags);
+}
+
+static struct virt_dma_lockops inband_lock_ops = {
+	.init			= inband_init_chan_lock,
+	.lock			= inband_lock_chan,
+	.unlock			= inband_unlock_chan,
+	.lock_irq		= inband_lock_irq_chan,
+	.unlock_irq		= inband_unlock_irq_chan,
+	.lock_irqsave		= inband_lock_irqsave_chan,
+	.unlock_irqrestore	= inband_unlock_irqrestore_chan,
+};
+
+static void oob_init_chan_lock(struct virt_dma_chan *vc)
+{
+	raw_spin_lock_init(&vc->oob_lock);
+}
+
+static void oob_lock_chan(struct virt_dma_chan *vc)
+{
+	raw_spin_lock(&vc->oob_lock);
+}
+
+static void oob_unlock_chan(struct virt_dma_chan *vc)
+{
+	raw_spin_unlock(&vc->oob_lock);
+}
+
+static void oob_lock_irq_chan(struct virt_dma_chan *vc)
+{
+	raw_spin_lock_irq(&vc->oob_lock);
+}
+
+static void oob_unlock_irq_chan(struct virt_dma_chan *vc)
+{
+	raw_spin_unlock_irq(&vc->oob_lock);
+}
+
+static unsigned long oob_lock_irqsave_chan(struct virt_dma_chan *vc)
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&vc->oob_lock, flags);
+
+	return flags;
+}
+
+static void oob_unlock_irqrestore_chan(struct virt_dma_chan *vc,
+				unsigned long flags)
+{
+	raw_spin_unlock_irqrestore(&vc->oob_lock, flags);
+}
+
+static struct virt_dma_lockops oob_lock_ops = {
+	.init			= oob_init_chan_lock,
+	.lock			= oob_lock_chan,
+	.unlock			= oob_unlock_chan,
+	.lock_irq		= oob_lock_irq_chan,
+	.unlock_irq		= oob_unlock_irq_chan,
+	.lock_irqsave		= oob_lock_irqsave_chan,
+	.unlock_irqrestore	= oob_unlock_irqrestore_chan,
+};
+
+#endif
+
 void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev)
 {
 	dma_cookie_init(&vc->chan);
 
-	spin_lock_init(&vc->lock);
+#ifdef CONFIG_DMA_VIRTUAL_CHANNELS_OOB
+	vc->lock_ops = test_bit(DMA_OOB, dmadev->cap_mask.bits) ?
+		&oob_lock_ops : &inband_lock_ops;
+#endif
+	vchan_lock_init(vc);
 	INIT_LIST_HEAD(&vc->desc_allocated);
 	INIT_LIST_HEAD(&vc->desc_submitted);
 	INIT_LIST_HEAD(&vc->desc_issued);
diff --git a/kernel/drivers/dma/virt-dma.h b/kernel/drivers/dma/virt-dma.h
index e9f5250fbe..5e01bc89ff 100644
--- a/kernel/drivers/dma/virt-dma.h
+++ b/kernel/drivers/dma/virt-dma.h
@@ -19,12 +19,22 @@ struct virt_dma_desc {
 	struct list_head node;
 };
 
+struct virt_dma_lockops;
+
 struct virt_dma_chan {
 	struct dma_chan	chan;
 	struct tasklet_struct task;
 	void (*desc_free)(struct virt_dma_desc *);
 
+#ifdef CONFIG_DMA_VIRTUAL_CHANNELS_OOB
+	struct virt_dma_lockops *lock_ops;
+	union {
+		spinlock_t lock;
+		hard_spinlock_t oob_lock;
+	};
+#else
 	spinlock_t lock;
+#endif
 
 	/* protected by vc.lock */
 	struct list_head desc_allocated;
@@ -41,6 +51,107 @@ static inline struct virt_dma_chan *to_virt_chan(struct dma_chan *chan)
 	return container_of(chan, struct virt_dma_chan, chan);
 }
 
+#ifdef CONFIG_DMA_VIRTUAL_CHANNELS_OOB
+
+struct virt_dma_lockops {
+	void (*init)(struct virt_dma_chan *vc);
+	void (*lock)(struct virt_dma_chan *vc);
+	void (*unlock)(struct virt_dma_chan *vc);
+	void (*lock_irq)(struct virt_dma_chan *vc);
+	void (*unlock_irq)(struct virt_dma_chan *vc);
+	unsigned long (*lock_irqsave)(struct virt_dma_chan *vc);
+	void (*unlock_irqrestore)(struct virt_dma_chan *vc,
+				unsigned long flags);
+};
+
+static inline void vchan_lock_init(struct virt_dma_chan *vc)
+{
+	vc->lock_ops->init(vc);
+}
+
+static inline void vchan_lock(struct virt_dma_chan *vc)
+{
+	vc->lock_ops->lock(vc);
+}
+
+static inline void vchan_unlock(struct virt_dma_chan *vc)
+{
+	vc->lock_ops->unlock(vc);
+}
+
+static inline void vchan_lock_irq(struct virt_dma_chan *vc)
+{
+	vc->lock_ops->lock_irq(vc);
+}
+
+static inline void vchan_unlock_irq(struct virt_dma_chan *vc)
+{
+	vc->lock_ops->unlock_irq(vc);
+}
+
+static inline
+unsigned long __vchan_lock_irqsave(struct virt_dma_chan *vc)
+{
+	return vc->lock_ops->lock_irqsave(vc);
+}
+
+#define vchan_lock_irqsave(__vc, __flags)		\
+	do {						\
+		(__flags) = __vchan_lock_irqsave(__vc);	\
+	} while (0)
+
+static inline
+void vchan_unlock_irqrestore(struct virt_dma_chan *vc,
+			unsigned long flags)
+{
+	vc->lock_ops->unlock_irqrestore(vc, flags);
+}
+
+static inline bool vchan_oob_handled(struct virt_dma_desc *vd)
+{
+	return !!(vd->tx.flags & DMA_OOB_INTERRUPT);
+}
+
+static inline bool vchan_oob_pulsed(struct virt_dma_desc *vd)
+{
+	return !!(vd->tx.flags & DMA_OOB_PULSE);
+}
+
+#else
+
+#define vchan_lock_init(__vc)				\
+	spin_lock_init(&(__vc)->lock)
+
+#define vchan_lock(__vc)				\
+	spin_lock(&(__vc)->lock)
+
+#define vchan_unlock(__vc)				\
+	spin_unlock(&(__vc)->lock)
+
+#define vchan_lock_irq(__vc)				\
+	spin_lock_irq(&(__vc)->lock)
+
+#define vchan_unlock_irq(__vc)				\
+	spin_unlock_irq(&(__vc)->lock)
+
+#define vchan_lock_irqsave(__vc, __flags)		\
+	spin_lock_irqsave(&(__vc)->lock, __flags)
+
+#define vchan_unlock_irqrestore(__vc, __flags)		\
+	spin_unlock_irqrestore(&(__vc)->lock, __flags)
+
+static inline bool vchan_oob_handled(struct virt_dma_desc *vd)
+{
+	return false;
+}
+
+static inline bool vchan_oob_pulsed(struct virt_dma_desc *vd)
+{
+	return false;
+}
+
+#endif	/* !CONFIG_DMA_VIRTUAL_CHANNELS_OOB */
+
 void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head);
 void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev);
 struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *, dma_cookie_t);
@@ -66,9 +177,9 @@ static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan
 	vd->tx_result.result = DMA_TRANS_NOERROR;
 	vd->tx_result.residue = 0;
 
-	spin_lock_irqsave(&vc->lock, flags);
+	vchan_lock_irqsave(vc, flags);
 	list_add_tail(&vd->node, &vc->desc_allocated);
-	spin_unlock_irqrestore(&vc->lock, flags);
+	vchan_unlock_irqrestore(vc, flags);
 
 	return &vd->tx;
 }
@@ -116,9 +227,9 @@ static inline void vchan_vdesc_fini(struct virt_dma_desc *vd)
 	if (dmaengine_desc_test_reuse(&vd->tx)) {
 		unsigned long flags;
 
-		spin_lock_irqsave(&vc->lock, flags);
+		vchan_lock_irqsave(vc, flags);
 		list_add(&vd->node, &vc->desc_allocated);
-		spin_unlock_irqrestore(&vc->lock, flags);
+		vchan_unlock_irqrestore(vc, flags);
 	} else {
 		vc->desc_free(vd);
 	}
@@ -190,11 +301,11 @@ static inline void vchan_free_chan_resources(struct virt_dma_chan *vc)
 	unsigned long flags;
 	LIST_HEAD(head);
 
-	spin_lock_irqsave(&vc->lock, flags);
+	vchan_lock_irqsave(vc, flags);
 	vchan_get_all_descriptors(vc, &head);
 	list_for_each_entry(vd, &head, node)
 		dmaengine_desc_clear_reuse(&vd->tx);
-	spin_unlock_irqrestore(&vc->lock, flags);
+	vchan_unlock_irqrestore(vc, flags);
 
 	vchan_dma_desc_free_list(vc, &head);
 }
@@ -215,11 +326,11 @@ static inline void vchan_synchronize(struct virt_dma_chan *vc)
 
 	tasklet_kill(&vc->task);
 
-	spin_lock_irqsave(&vc->lock, flags);
+	vchan_lock_irqsave(vc, flags);
 
 	list_splice_tail_init(&vc->desc_terminated, &head);
 
-	spin_unlock_irqrestore(&vc->lock, flags);
+	vchan_unlock_irqrestore(vc, flags);
 
 	vchan_dma_desc_free_list(vc, &head);
 }
diff --git a/kernel/drivers/gpio/gpio-davinci.c b/kernel/drivers/gpio/gpio-davinci.c
index 6f21385037..b7ae10d6a8 100644
--- a/kernel/drivers/gpio/gpio-davinci.c
+++ b/kernel/drivers/gpio/gpio-davinci.c
@@ -326,7 +326,7 @@ static struct irq_chip gpio_irqchip = {
 	.irq_enable	= gpio_irq_enable,
 	.irq_disable	= gpio_irq_disable,
 	.irq_set_type	= gpio_irq_type,
-	.flags		= IRQCHIP_SET_TYPE_MASKED,
+	.flags		= IRQCHIP_SET_TYPE_MASKED | IRQCHIP_PIPELINE_SAFE,
 };
 
 static void gpio_irq_handler(struct irq_desc *desc)
diff --git a/kernel/drivers/gpio/gpio-mxc.c b/kernel/drivers/gpio/gpio-mxc.c
index ba6ed2a413..96d523f48d 100644
--- a/kernel/drivers/gpio/gpio-mxc.c
+++ b/kernel/drivers/gpio/gpio-mxc.c
@@ -361,7 +361,8 @@ static int mxc_gpio_init_gc(struct mxc_gpio_port *port, int irq_base)
 	ct->chip.irq_unmask = irq_gc_mask_set_bit;
 	ct->chip.irq_set_type = gpio_set_irq_type;
 	ct->chip.irq_set_wake = gpio_set_wake_irq;
-	ct->chip.flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND;
+	ct->chip.flags = IRQCHIP_MASK_ON_SUSPEND |
+		IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND | IRQCHIP_PIPELINE_SAFE;
 	ct->regs.ack = GPIO_ISR;
 	ct->regs.mask = GPIO_IMR;
 
diff --git a/kernel/drivers/gpio/gpio-omap.c b/kernel/drivers/gpio/gpio-omap.c
index a7e8ed5191..3316893dec 100644
--- a/kernel/drivers/gpio/gpio-omap.c
+++ b/kernel/drivers/gpio/gpio-omap.c
@@ -55,7 +55,7 @@ struct gpio_bank {
 	u32 saved_datain;
 	u32 level_mask;
 	u32 toggle_mask;
-	raw_spinlock_t lock;
+	hard_spinlock_t lock;
 	raw_spinlock_t wa_lock;
 	struct gpio_chip chip;
 	struct clk *dbck;
@@ -1058,7 +1058,7 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc)
 
 	ret = devm_request_irq(bank->chip.parent, bank->irq,
 			       omap_gpio_irq_handler,
-			       0, dev_name(bank->chip.parent), bank);
+			       IRQF_OOB, dev_name(bank->chip.parent), bank);
 	if (ret)
 		gpiochip_remove(&bank->chip);
 
@@ -1406,7 +1406,7 @@ static int omap_gpio_probe(struct platform_device *pdev)
 	irqc->irq_bus_lock = omap_gpio_irq_bus_lock,
 	irqc->irq_bus_sync_unlock = gpio_irq_bus_sync_unlock,
 	irqc->name = dev_name(&pdev->dev);
-	irqc->flags = IRQCHIP_MASK_ON_SUSPEND;
+	irqc->flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_PIPELINE_SAFE;
 	irqc->parent_device = dev;
 
 	bank->irq = platform_get_irq(pdev, 0);
diff --git a/kernel/drivers/gpio/gpio-pca953x.c b/kernel/drivers/gpio/gpio-pca953x.c
index 3ad1a9e432..cad58d0e51 100644
--- a/kernel/drivers/gpio/gpio-pca953x.c
+++ b/kernel/drivers/gpio/gpio-pca953x.c
@@ -855,6 +855,7 @@ static int pca953x_irq_setup(struct pca953x_chip *chip, int irq_base)
 	irq_chip->irq_bus_sync_unlock = pca953x_irq_bus_sync_unlock;
 	irq_chip->irq_set_type = pca953x_irq_set_type;
 	irq_chip->irq_shutdown = pca953x_irq_shutdown;
+	irq_chip->flags |= IRQCHIP_PIPELINE_SAFE;
 
 	girq = &chip->gpio_chip.irq;
 	girq->chip = irq_chip;
diff --git a/kernel/drivers/gpio/gpio-pl061.c b/kernel/drivers/gpio/gpio-pl061.c
index f1b53dd1df..c890b9cfd1 100644
--- a/kernel/drivers/gpio/gpio-pl061.c
+++ b/kernel/drivers/gpio/gpio-pl061.c
@@ -48,7 +48,7 @@ struct pl061_context_save_regs {
 #endif
 
 struct pl061 {
-	raw_spinlock_t		lock;
+	hard_spinlock_t		lock;
 
 	void __iomem		*base;
 	struct gpio_chip	gc;
@@ -321,6 +321,7 @@ static int pl061_probe(struct amba_device *adev, const struct amba_id *id)
 	pl061->irq_chip.irq_unmask = pl061_irq_unmask;
 	pl061->irq_chip.irq_set_type = pl061_irq_type;
 	pl061->irq_chip.irq_set_wake = pl061_irq_set_wake;
+	pl061->irq_chip.flags = IRQCHIP_PIPELINE_SAFE;
 
 	writeb(0, pl061->base + GPIOIE); /* disable irqs */
 	irq = adev->irq[0];
diff --git a/kernel/drivers/gpio/gpio-xilinx.c b/kernel/drivers/gpio/gpio-xilinx.c
index 67f9f82e0d..33f03ace9c 100644
--- a/kernel/drivers/gpio/gpio-xilinx.c
+++ b/kernel/drivers/gpio/gpio-xilinx.c
@@ -45,7 +45,7 @@ struct xgpio_instance {
 	unsigned int gpio_width[2];
 	u32 gpio_state[2];
 	u32 gpio_dir[2];
-	spinlock_t gpio_lock[2];
+	hard_spinlock_t gpio_lock[2];
 };
 
 static inline int xgpio_index(struct xgpio_instance *chip, int gpio)
@@ -110,7 +110,7 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 	int index =  xgpio_index(chip, gpio);
 	int offset =  xgpio_offset(chip, gpio);
 
-	spin_lock_irqsave(&chip->gpio_lock[index], flags);
+	raw_spin_lock_irqsave(&chip->gpio_lock[index], flags);
 
 	/* Write to GPIO signal and set its direction to output */
 	if (val)
@@ -121,7 +121,7 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 	xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET +
 		       xgpio_regoffset(chip, gpio), chip->gpio_state[index]);
 
-	spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
+	raw_spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
 }
 
 /**
@@ -141,7 +141,7 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask,
 	int index = xgpio_index(chip, 0);
 	int offset, i;
 
-	spin_lock_irqsave(&chip->gpio_lock[index], flags);
+	raw_spin_lock_irqsave(&chip->gpio_lock[index], flags);
 
 	/* Write to GPIO signals */
 	for (i = 0; i < gc->ngpio; i++) {
@@ -152,9 +152,9 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask,
 			xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET +
 				       index * XGPIO_CHANNEL_OFFSET,
 				       chip->gpio_state[index]);
-			spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
+			raw_spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
 			index =  xgpio_index(chip, i);
-			spin_lock_irqsave(&chip->gpio_lock[index], flags);
+			raw_spin_lock_irqsave(&chip->gpio_lock[index], flags);
 		}
 		if (__test_and_clear_bit(i, mask)) {
 			offset =  xgpio_offset(chip, i);
@@ -168,7 +168,7 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask,
 	xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET +
 		       index * XGPIO_CHANNEL_OFFSET, chip->gpio_state[index]);
 
-	spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
+	raw_spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
 }
 
 /**
@@ -187,14 +187,14 @@ static int xgpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
 	int index =  xgpio_index(chip, gpio);
 	int offset =  xgpio_offset(chip, gpio);
 
-	spin_lock_irqsave(&chip->gpio_lock[index], flags);
+	raw_spin_lock_irqsave(&chip->gpio_lock[index], flags);
 
 	/* Set the GPIO bit in shadow register and set direction as input */
 	chip->gpio_dir[index] |= BIT(offset);
 	xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET +
 		       xgpio_regoffset(chip, gpio), chip->gpio_dir[index]);
 
-	spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
+	raw_spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
 
 	return 0;
 }
@@ -218,7 +218,7 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 	int index =  xgpio_index(chip, gpio);
 	int offset =  xgpio_offset(chip, gpio);
 
-	spin_lock_irqsave(&chip->gpio_lock[index], flags);
+	raw_spin_lock_irqsave(&chip->gpio_lock[index], flags);
 
 	/* Write state of GPIO signal */
 	if (val)
@@ -233,7 +233,7 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val)
 	xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET +
 			xgpio_regoffset(chip, gpio), chip->gpio_dir[index]);
 
-	spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
+	raw_spin_unlock_irqrestore(&chip->gpio_lock[index], flags);
 
 	return 0;
 }
@@ -291,7 +291,7 @@ static int xgpio_probe(struct platform_device *pdev)
 	if (of_property_read_u32(np, "xlnx,gpio-width", &chip->gpio_width[0]))
 		chip->gpio_width[0] = 32;
 
-	spin_lock_init(&chip->gpio_lock[0]);
+	raw_spin_lock_init(&chip->gpio_lock[0]);
 
 	if (of_property_read_u32(np, "xlnx,is-dual", &is_dual))
 		is_dual = 0;
@@ -314,7 +314,7 @@ static int xgpio_probe(struct platform_device *pdev)
 					 &chip->gpio_width[1]))
 			chip->gpio_width[1] = 32;
 
-		spin_lock_init(&chip->gpio_lock[1]);
+		raw_spin_lock_init(&chip->gpio_lock[1]);
 	}
 
 	chip->gc.base = -1;
diff --git a/kernel/drivers/gpio/gpio-zynq.c b/kernel/drivers/gpio/gpio-zynq.c
index c288a7502d..28c02802a0 100644
--- a/kernel/drivers/gpio/gpio-zynq.c
+++ b/kernel/drivers/gpio/gpio-zynq.c
@@ -601,7 +601,7 @@ static struct irq_chip zynq_gpio_level_irqchip = {
 	.irq_request_resources = zynq_gpio_irq_reqres,
 	.irq_release_resources = zynq_gpio_irq_relres,
 	.flags		= IRQCHIP_EOI_THREADED | IRQCHIP_EOI_IF_HANDLED |
-			  IRQCHIP_MASK_ON_SUSPEND,
+			  IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct irq_chip zynq_gpio_edge_irqchip = {
@@ -614,7 +614,7 @@ static struct irq_chip zynq_gpio_edge_irqchip = {
 	.irq_set_wake	= zynq_gpio_set_wake,
 	.irq_request_resources = zynq_gpio_irq_reqres,
 	.irq_release_resources = zynq_gpio_irq_relres,
-	.flags		= IRQCHIP_MASK_ON_SUSPEND,
+	.flags		= IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_PIPELINE_SAFE,
 };
 
 static void zynq_gpio_handle_bank_irq(struct zynq_gpio *gpio,
diff --git a/kernel/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c b/kernel/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c
index 96f3908e4c..e82dda7e78 100644
--- a/kernel/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c
+++ b/kernel/drivers/gpu/drm/msm/disp/dpu1/dpu_mdss.c
@@ -94,6 +94,7 @@ static struct irq_chip dpu_mdss_irq_chip = {
 	.name = "dpu_mdss",
 	.irq_mask = dpu_mdss_irq_mask,
 	.irq_unmask = dpu_mdss_irq_unmask,
+	.flags = IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct lock_class_key dpu_mdss_lock_key, dpu_mdss_request_key;
diff --git a/kernel/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c b/kernel/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c
index 09bd46ad82..781d701dfc 100644
--- a/kernel/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c
+++ b/kernel/drivers/gpu/drm/msm/disp/mdp5/mdp5_mdss.c
@@ -91,6 +91,7 @@ static struct irq_chip mdss_hw_irq_chip = {
 	.name		= "mdss",
 	.irq_mask	= mdss_hw_mask_irq,
 	.irq_unmask	= mdss_hw_unmask_irq,
+	.flags		= IRQCHIP_PIPELINE_SAFE,
 };
 
 static int mdss_hw_irqdomain_map(struct irq_domain *d, unsigned int irq,
@@ -254,7 +255,7 @@ int mdp5_mdss_init(struct drm_device *dev)
 	}
 
 	ret = devm_request_irq(dev->dev, platform_get_irq(pdev, 0),
-			       mdss_irq, 0, "mdss_isr", mdp5_mdss);
+			       mdss_irq, IRQF_OOB, "mdss_isr", mdp5_mdss);
 	if (ret) {
 		DRM_DEV_ERROR(dev->dev, "failed to init irq: %d\n", ret);
 		goto fail_irq;
diff --git a/kernel/drivers/gpu/ipu-v3/ipu-common.c b/kernel/drivers/gpu/ipu-v3/ipu-common.c
index d166ee262c..6f4db392ac 100644
--- a/kernel/drivers/gpu/ipu-v3/ipu-common.c
+++ b/kernel/drivers/gpu/ipu-v3/ipu-common.c
@@ -1238,6 +1238,7 @@ static int ipu_irq_init(struct ipu_soc *ipu)
 		ct->chip.irq_ack = irq_gc_ack_set_bit;
 		ct->chip.irq_mask = irq_gc_mask_clr_bit;
 		ct->chip.irq_unmask = irq_gc_mask_set_bit;
+		ct->chip.flags = IRQCHIP_PIPELINE_SAFE;
 		ct->regs.ack = IPU_INT_STAT(i / 32);
 		ct->regs.mask = IPU_INT_CTRL(i / 32);
 	}
diff --git a/kernel/drivers/iio/industrialio-trigger.c b/kernel/drivers/iio/industrialio-trigger.c
index 6bcc562d78..e9172bb275 100644
--- a/kernel/drivers/iio/industrialio-trigger.c
+++ b/kernel/drivers/iio/industrialio-trigger.c
@@ -544,6 +544,7 @@ struct iio_trigger *viio_trigger_alloc(const char *fmt, va_list vargs)
 	trig->subirq_chip.name = trig->name;
 	trig->subirq_chip.irq_mask = &iio_trig_subirqmask;
 	trig->subirq_chip.irq_unmask = &iio_trig_subirqunmask;
+	trig->subirq_chip.flags = IRQCHIP_PIPELINE_SAFE;
 	for (i = 0; i < CONFIG_IIO_CONSUMERS_PER_TRIGGER; i++) {
 		irq_set_chip(trig->subirq_base + i, &trig->subirq_chip);
 		irq_set_handler(trig->subirq_base + i, &handle_simple_irq);
diff --git a/kernel/drivers/irqchip/exynos-combiner.c b/kernel/drivers/irqchip/exynos-combiner.c
index 0b85d9a3fb..504bdd05f4 100644
--- a/kernel/drivers/irqchip/exynos-combiner.c
+++ b/kernel/drivers/irqchip/exynos-combiner.c
@@ -24,7 +24,7 @@
 
 #define IRQ_IN_COMBINER		8
 
-static DEFINE_SPINLOCK(irq_controller_lock);
+static DEFINE_HARD_SPINLOCK(irq_controller_lock);
 
 struct combiner_chip_data {
 	unsigned int hwirq_offset;
@@ -71,9 +71,9 @@ static void combiner_handle_cascade_irq(struct irq_desc *desc)
 
 	chained_irq_enter(chip, desc);
 
-	spin_lock(&irq_controller_lock);
+	raw_spin_lock(&irq_controller_lock);
 	status = readl_relaxed(chip_data->base + COMBINER_INT_STATUS);
-	spin_unlock(&irq_controller_lock);
+	raw_spin_unlock(&irq_controller_lock);
 	status &= chip_data->irq_mask;
 
 	if (status == 0)
@@ -113,6 +113,7 @@ static struct irq_chip combiner_chip = {
 #ifdef CONFIG_SMP
 	.irq_set_affinity	= combiner_set_affinity,
 #endif
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static void __init combiner_cascade_irq(struct combiner_chip_data *combiner_data,
diff --git a/kernel/drivers/irqchip/irq-bcm2835.c b/kernel/drivers/irqchip/irq-bcm2835.c
index a1e004af23..bbf1cdb952 100644
--- a/kernel/drivers/irqchip/irq-bcm2835.c
+++ b/kernel/drivers/irqchip/irq-bcm2835.c
@@ -102,7 +102,8 @@ static void armctrl_unmask_irq(struct irq_data *d)
 static struct irq_chip armctrl_chip = {
 	.name = "ARMCTRL-level",
 	.irq_mask = armctrl_mask_irq,
-	.irq_unmask = armctrl_unmask_irq
+	.irq_unmask = armctrl_unmask_irq,
+	.flags = IRQCHIP_PIPELINE_SAFE,
 };
 
 static int armctrl_xlate(struct irq_domain *d, struct device_node *ctrlr,
diff --git a/kernel/drivers/irqchip/irq-bcm2836.c b/kernel/drivers/irqchip/irq-bcm2836.c
index cbc7c740e4..89b47fc43a 100644
--- a/kernel/drivers/irqchip/irq-bcm2836.c
+++ b/kernel/drivers/irqchip/irq-bcm2836.c
@@ -58,6 +58,7 @@ static struct irq_chip bcm2836_arm_irqchip_timer = {
 	.name		= "bcm2836-timer",
 	.irq_mask	= bcm2836_arm_irqchip_mask_timer_irq,
 	.irq_unmask	= bcm2836_arm_irqchip_unmask_timer_irq,
+	.flags		= IRQCHIP_PIPELINE_SAFE,
 };
 
 static void bcm2836_arm_irqchip_mask_pmu_irq(struct irq_data *d)
@@ -74,6 +75,7 @@ static struct irq_chip bcm2836_arm_irqchip_pmu = {
 	.name		= "bcm2836-pmu",
 	.irq_mask	= bcm2836_arm_irqchip_mask_pmu_irq,
 	.irq_unmask	= bcm2836_arm_irqchip_unmask_pmu_irq,
+	.flags		= IRQCHIP_PIPELINE_SAFE,
 };
 
 static void bcm2836_arm_irqchip_mask_gpu_irq(struct irq_data *d)
@@ -88,6 +90,7 @@ static struct irq_chip bcm2836_arm_irqchip_gpu = {
 	.name		= "bcm2836-gpu",
 	.irq_mask	= bcm2836_arm_irqchip_mask_gpu_irq,
 	.irq_unmask	= bcm2836_arm_irqchip_unmask_gpu_irq,
+	.flags		= IRQCHIP_PIPELINE_SAFE,
 };
 
 static void bcm2836_arm_irqchip_dummy_op(struct irq_data *d)
diff --git a/kernel/drivers/irqchip/irq-gic-v2m.c b/kernel/drivers/irqchip/irq-gic-v2m.c
index fbec07d634..c3f19ef14c 100644
--- a/kernel/drivers/irqchip/irq-gic-v2m.c
+++ b/kernel/drivers/irqchip/irq-gic-v2m.c
@@ -89,6 +89,7 @@ static struct irq_chip gicv2m_msi_irq_chip = {
 	.irq_unmask		= gicv2m_unmask_msi_irq,
 	.irq_eoi		= irq_chip_eoi_parent,
 	.irq_write_msi_msg	= pci_msi_domain_write_msg,
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct msi_domain_info gicv2m_msi_domain_info = {
@@ -130,6 +131,7 @@ static struct irq_chip gicv2m_irq_chip = {
 	.irq_eoi		= irq_chip_eoi_parent,
 	.irq_set_affinity	= irq_chip_set_affinity_parent,
 	.irq_compose_msi_msg	= gicv2m_compose_msi_msg,
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static int gicv2m_irq_gic_domain_alloc(struct irq_domain *domain,
@@ -252,6 +254,7 @@ static bool is_msi_spi_valid(u32 base, u32 num)
 
 static struct irq_chip gicv2m_pmsi_irq_chip = {
 	.name			= "pMSI",
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct msi_domain_ops gicv2m_pmsi_ops = {
diff --git a/kernel/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c b/kernel/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
index 634263dfd7..0b4b81a978 100644
--- a/kernel/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
+++ b/kernel/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
@@ -22,7 +22,8 @@ static struct irq_chip its_msi_irq_chip = {
 	.irq_mask = irq_chip_mask_parent,
 	.irq_unmask = irq_chip_unmask_parent,
 	.irq_eoi = irq_chip_eoi_parent,
-	.irq_set_affinity = msi_domain_set_affinity
+	.irq_set_affinity = msi_domain_set_affinity,
+	.flags = IRQCHIP_PIPELINE_SAFE,
 };
 
 static u32 fsl_mc_msi_domain_get_msi_id(struct irq_domain *domain,
diff --git a/kernel/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/kernel/drivers/irqchip/irq-gic-v3-its-pci-msi.c
index 87711e0f80..a148d0d200 100644
--- a/kernel/drivers/irqchip/irq-gic-v3-its-pci-msi.c
+++ b/kernel/drivers/irqchip/irq-gic-v3-its-pci-msi.c
@@ -29,6 +29,7 @@ static struct irq_chip its_msi_irq_chip = {
 	.irq_mask		= its_mask_msi_irq,
 	.irq_eoi		= irq_chip_eoi_parent,
 	.irq_write_msi_msg	= pci_msi_domain_write_msg,
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static int its_pci_msi_vec_count(struct pci_dev *pdev, void *data)
diff --git a/kernel/drivers/irqchip/irq-gic-v3-its-platform-msi.c b/kernel/drivers/irqchip/irq-gic-v3-its-platform-msi.c
index daa6d5053b..ae29443e97 100644
--- a/kernel/drivers/irqchip/irq-gic-v3-its-platform-msi.c
+++ b/kernel/drivers/irqchip/irq-gic-v3-its-platform-msi.c
@@ -12,6 +12,7 @@
 
 static struct irq_chip its_pmsi_irq_chip = {
 	.name			= "ITS-pMSI",
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static int of_pmsi_get_dev_id(struct irq_domain *domain, struct device *dev,
diff --git a/kernel/drivers/irqchip/irq-gic-v3-mbi.c b/kernel/drivers/irqchip/irq-gic-v3-mbi.c
index e81e89a81c..b213cd755d 100644
--- a/kernel/drivers/irqchip/irq-gic-v3-mbi.c
+++ b/kernel/drivers/irqchip/irq-gic-v3-mbi.c
@@ -216,7 +216,7 @@ static struct irq_chip mbi_pmsi_irq_chip = {
 	.name			= "pMSI",
 	.irq_set_type		= irq_chip_set_type_parent,
 	.irq_compose_msi_msg	= mbi_compose_mbi_msg,
-	.flags			= IRQCHIP_SUPPORTS_LEVEL_MSI,
+	.flags			= IRQCHIP_SUPPORTS_LEVEL_MSI | IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct msi_domain_ops mbi_pmsi_ops = {
diff --git a/kernel/drivers/irqchip/irq-gic-v3.c b/kernel/drivers/irqchip/irq-gic-v3.c
index 6db1dbc6a7..128a69fdda 100644
--- a/kernel/drivers/irqchip/irq-gic-v3.c
+++ b/kernel/drivers/irqchip/irq-gic-v3.c
@@ -1356,7 +1356,8 @@ static struct irq_chip gic_chip = {
 	.ipi_send_mask		= gic_ipi_send_mask,
 	.flags			= IRQCHIP_SET_TYPE_MASKED |
 				  IRQCHIP_SKIP_SET_WAKE |
-				  IRQCHIP_MASK_ON_SUSPEND,
+				  IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct irq_chip gic_eoimode1_chip = {
@@ -1375,7 +1376,8 @@ static struct irq_chip gic_eoimode1_chip = {
 	.ipi_send_mask		= gic_ipi_send_mask,
 	.flags			= IRQCHIP_SET_TYPE_MASKED |
 				  IRQCHIP_SKIP_SET_WAKE |
-				  IRQCHIP_MASK_ON_SUSPEND,
+				  IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_PIPELINE_SAFE,
 };
 
 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
diff --git a/kernel/drivers/irqchip/irq-gic.c b/kernel/drivers/irqchip/irq-gic.c
index ac027de9e5..db7598ab82 100644
--- a/kernel/drivers/irqchip/irq-gic.c
+++ b/kernel/drivers/irqchip/irq-gic.c
@@ -87,7 +87,7 @@ struct gic_chip_data {
 
 #ifdef CONFIG_BL_SWITCHER
 
-static DEFINE_RAW_SPINLOCK(cpu_map_lock);
+static DEFINE_HARD_SPINLOCK(cpu_map_lock);
 
 #define gic_lock_irqsave(f)		\
 	raw_spin_lock_irqsave(&cpu_map_lock, (f))
@@ -416,7 +416,8 @@ static const struct irq_chip gic_chip = {
 	.irq_set_irqchip_state	= gic_irq_set_irqchip_state,
 	.flags			= IRQCHIP_SET_TYPE_MASKED |
 				  IRQCHIP_SKIP_SET_WAKE |
-				  IRQCHIP_MASK_ON_SUSPEND,
+				  IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_PIPELINE_SAFE,
 };
 
 void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
diff --git a/kernel/drivers/irqchip/irq-imx-irqsteer.c b/kernel/drivers/irqchip/irq-imx-irqsteer.c
index 1edf7692a7..54a576a151 100644
--- a/kernel/drivers/irqchip/irq-imx-irqsteer.c
+++ b/kernel/drivers/irqchip/irq-imx-irqsteer.c
@@ -29,7 +29,7 @@ struct irqsteer_data {
 	struct clk		*ipg_clk;
 	int			irq[CHAN_MAX_OUTPUT_INT];
 	int			irq_count;
-	raw_spinlock_t		lock;
+	hard_spinlock_t		lock;
 	int			reg_num;
 	int			channel;
 	struct irq_domain	*domain;
@@ -74,6 +74,7 @@ static struct irq_chip imx_irqsteer_irq_chip = {
 	.name		= "irqsteer",
 	.irq_mask	= imx_irqsteer_irq_mask,
 	.irq_unmask	= imx_irqsteer_irq_unmask,
+	.flags		= IRQCHIP_PIPELINE_SAFE,
 };
 
 static int imx_irqsteer_irq_map(struct irq_domain *h, unsigned int irq,
diff --git a/kernel/drivers/irqchip/irq-omap-intc.c b/kernel/drivers/irqchip/irq-omap-intc.c
index d360a6eddd..a8647443cc 100644
--- a/kernel/drivers/irqchip/irq-omap-intc.c
+++ b/kernel/drivers/irqchip/irq-omap-intc.c
@@ -211,7 +211,7 @@ static int __init omap_alloc_gc_of(struct irq_domain *d, void __iomem *base)
 		ct->chip.irq_mask = irq_gc_mask_disable_reg;
 		ct->chip.irq_unmask = irq_gc_unmask_enable_reg;
 
-		ct->chip.flags |= IRQCHIP_SKIP_SET_WAKE;
+		ct->chip.flags |= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_PIPELINE_SAFE;
 
 		ct->regs.enable = INTC_MIR_CLEAR0 + 32 * i;
 		ct->regs.disable = INTC_MIR_SET0 + 32 * i;
diff --git a/kernel/drivers/irqchip/irq-sun4i.c b/kernel/drivers/irqchip/irq-sun4i.c
index fb78d66235..4f5c42be18 100644
--- a/kernel/drivers/irqchip/irq-sun4i.c
+++ b/kernel/drivers/irqchip/irq-sun4i.c
@@ -87,7 +87,7 @@ static struct irq_chip sun4i_irq_chip = {
 	.irq_eoi	= sun4i_irq_ack,
 	.irq_mask	= sun4i_irq_mask,
 	.irq_unmask	= sun4i_irq_unmask,
-	.flags		= IRQCHIP_EOI_THREADED | IRQCHIP_EOI_IF_HANDLED,
+	.flags		= IRQCHIP_EOI_THREADED | IRQCHIP_EOI_IF_HANDLED | IRQCHIP_PIPELINE_SAFE,
 };
 
 static int sun4i_irq_map(struct irq_domain *d, unsigned int virq,
diff --git a/kernel/drivers/irqchip/irq-sunxi-nmi.c b/kernel/drivers/irqchip/irq-sunxi-nmi.c
index a412b5d5d0..ada16f4088 100644
--- a/kernel/drivers/irqchip/irq-sunxi-nmi.c
+++ b/kernel/drivers/irqchip/irq-sunxi-nmi.c
@@ -200,7 +200,9 @@ static int __init sunxi_sc_nmi_irq_init(struct device_node *node,
 	gc->chip_types[0].chip.irq_unmask	= irq_gc_mask_set_bit;
 	gc->chip_types[0].chip.irq_eoi		= irq_gc_ack_set_bit;
 	gc->chip_types[0].chip.irq_set_type	= sunxi_sc_nmi_set_type;
-	gc->chip_types[0].chip.flags		= IRQCHIP_EOI_THREADED | IRQCHIP_EOI_IF_HANDLED;
+	gc->chip_types[0].chip.flags		= IRQCHIP_EOI_THREADED |
+									IRQCHIP_EOI_IF_HANDLED |
+									IRQCHIP_PIPELINE_SAFE;
 	gc->chip_types[0].regs.ack		= reg_offs->pend;
 	gc->chip_types[0].regs.mask		= reg_offs->enable;
 	gc->chip_types[0].regs.type		= reg_offs->ctrl;
diff --git a/kernel/drivers/irqchip/irq-ti-sci-inta.c b/kernel/drivers/irqchip/irq-ti-sci-inta.c
index 532d0ae172..eec751bdc1 100644
--- a/kernel/drivers/irqchip/irq-ti-sci-inta.c
+++ b/kernel/drivers/irqchip/irq-ti-sci-inta.c
@@ -262,6 +262,7 @@ static struct ti_sci_inta_vint_desc *ti_sci_inta_alloc_parent_irq(struct irq_dom
 	list_add_tail(&vint_desc->list, &inta->vint_list);
 	irq_set_chained_handler_and_data(vint_desc->parent_virq,
 					 ti_sci_inta_irq_handler, vint_desc);
+	irq_switch_oob(vint_desc->parent_virq, true);
 
 	return vint_desc;
 free_vint_desc:
@@ -543,6 +544,7 @@ static struct irq_chip ti_sci_inta_irq_chip = {
 	.irq_set_affinity	= ti_sci_inta_set_affinity,
 	.irq_request_resources	= ti_sci_inta_request_resources,
 	.irq_release_resources	= ti_sci_inta_release_resources,
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 /**
diff --git a/kernel/drivers/memory/omap-gpmc.c b/kernel/drivers/memory/omap-gpmc.c
index f80c2ea39c..ebcda04da4 100644
--- a/kernel/drivers/memory/omap-gpmc.c
+++ b/kernel/drivers/memory/omap-gpmc.c
@@ -1405,6 +1405,7 @@ static int gpmc_setup_irq(struct gpmc_device *gpmc)
 	gpmc->irq_chip.irq_mask = gpmc_irq_mask;
 	gpmc->irq_chip.irq_unmask = gpmc_irq_unmask;
 	gpmc->irq_chip.irq_set_type = gpmc_irq_set_type;
+	gpmc->irq_chip.flags = IRQCHIP_PIPELINE_SAFE;
 
 	gpmc_irq_domain = irq_domain_add_linear(gpmc->dev->of_node,
 						gpmc->nirqs,
@@ -1415,7 +1416,7 @@ static int gpmc_setup_irq(struct gpmc_device *gpmc)
 		return -ENODEV;
 	}
 
-	rc = request_irq(gpmc->irq, gpmc_handle_irq, 0, "gpmc", gpmc);
+	rc = request_irq(gpmc->irq, gpmc_handle_irq, IRQF_OOB, "gpmc", gpmc);
 	if (rc) {
 		dev_err(gpmc->dev, "failed to request irq %d: %d\n",
 			gpmc->irq, rc);
diff --git a/kernel/drivers/mfd/tps65217.c b/kernel/drivers/mfd/tps65217.c
index 2d9c282ec9..7987fea2d2 100644
--- a/kernel/drivers/mfd/tps65217.c
+++ b/kernel/drivers/mfd/tps65217.c
@@ -84,6 +84,7 @@ static struct irq_chip tps65217_irq_chip = {
 	.irq_bus_sync_unlock	= tps65217_irq_sync_unlock,
 	.irq_enable		= tps65217_irq_enable,
 	.irq_disable		= tps65217_irq_disable,
+	.flags			= IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct mfd_cell tps65217s[] = {
diff --git a/kernel/drivers/pci/controller/dwc/pcie-designware-host.c b/kernel/drivers/pci/controller/dwc/pcie-designware-host.c
index 1cb04f4e80..0b4c19fc39 100644
--- a/kernel/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/kernel/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -44,6 +44,7 @@ static struct irq_chip dw_pcie_msi_irq_chip = {
 	.irq_ack = dw_msi_ack_irq,
 	.irq_mask = dw_msi_mask_irq,
 	.irq_unmask = dw_msi_unmask_irq,
+	.flags = IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct msi_domain_info dw_pcie_msi_domain_info = {
diff --git a/kernel/drivers/pci/controller/pcie-brcmstb.c b/kernel/drivers/pci/controller/pcie-brcmstb.c
index 9c3d298224..452e2bb98f 100644
--- a/kernel/drivers/pci/controller/pcie-brcmstb.c
+++ b/kernel/drivers/pci/controller/pcie-brcmstb.c
@@ -457,6 +457,7 @@ static struct irq_chip brcm_msi_irq_chip = {
 	.irq_ack         = irq_chip_ack_parent,
 	.irq_mask        = pci_msi_mask_irq,
 	.irq_unmask      = pci_msi_unmask_irq,
+	.flags           = IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct msi_domain_info brcm_msi_domain_info = {
@@ -520,6 +521,7 @@ static struct irq_chip brcm_msi_bottom_irq_chip = {
 	.irq_compose_msi_msg	= brcm_msi_compose_msi_msg,
 	.irq_set_affinity	= brcm_msi_set_affinity,
 	.irq_ack                = brcm_msi_ack_irq,
+	.flags                  = IRQCHIP_PIPELINE_SAFE,
 };
 
 static int brcm_msi_alloc(struct brcm_msi *msi)
diff --git a/kernel/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/kernel/drivers/pinctrl/bcm/pinctrl-bcm2835.c
index 6768b2f03d..e37dd66a4e 100644
--- a/kernel/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+++ b/kernel/drivers/pinctrl/bcm/pinctrl-bcm2835.c
@@ -88,7 +88,7 @@ struct bcm2835_pinctrl {
 	struct pinctrl_desc pctl_desc;
 	struct pinctrl_gpio_range gpio_range;
 
-	raw_spinlock_t irq_lock[BCM2835_NUM_BANKS];
+	hard_spinlock_t irq_lock[BCM2835_NUM_BANKS];
 };
 
 /* pins are just named GPIO0..GPIO53 */
@@ -678,7 +678,7 @@ static struct irq_chip bcm2835_gpio_irq_chip = {
 	.irq_mask = bcm2835_gpio_irq_disable,
 	.irq_unmask = bcm2835_gpio_irq_enable,
 	.irq_set_wake = bcm2835_gpio_irq_set_wake,
-	.flags = IRQCHIP_MASK_ON_SUSPEND,
+	.flags = IRQCHIP_MASK_ON_SUSPEND|IRQCHIP_PIPELINE_SAFE,
 };
 
 static int bcm2835_pctl_get_groups_count(struct pinctrl_dev *pctldev)
diff --git a/kernel/drivers/pinctrl/intel/pinctrl-cherryview.c b/kernel/drivers/pinctrl/intel/pinctrl-cherryview.c
index 2ed17cdf94..80ee69a8cf 100644
--- a/kernel/drivers/pinctrl/intel/pinctrl-cherryview.c
+++ b/kernel/drivers/pinctrl/intel/pinctrl-cherryview.c
@@ -562,7 +562,7 @@ static const struct intel_pinctrl_soc_data *chv_soc_data[] = {
  * See Intel Atom Z8000 Processor Series Specification Update (Rev. 005),
  * errata #CHT34, for further information.
  */
-static DEFINE_RAW_SPINLOCK(chv_lock);
+static DEFINE_HARD_SPINLOCK(chv_lock);
 
 static u32 chv_pctrl_readl(struct intel_pinctrl *pctrl, unsigned int offset)
 {
@@ -1554,7 +1554,8 @@ static int chv_gpio_probe(struct intel_pinctrl *pctrl, int irq)
 	pctrl->irqchip.irq_mask = chv_gpio_irq_mask;
 	pctrl->irqchip.irq_unmask = chv_gpio_irq_unmask;
 	pctrl->irqchip.irq_set_type = chv_gpio_irq_type;
-	pctrl->irqchip.flags = IRQCHIP_SKIP_SET_WAKE;
+	pctrl->irqchip.flags = IRQCHIP_SKIP_SET_WAKE |
+				IRQCHIP_PIPELINE_SAFE;
 
 	chip->irq.chip = &pctrl->irqchip;
 	chip->irq.init_hw = chv_gpio_irq_init_hw;
diff --git a/kernel/drivers/pinctrl/qcom/pinctrl-msm.c b/kernel/drivers/pinctrl/qcom/pinctrl-msm.c
index a3cef80a78..c9e44528c7 100644
--- a/kernel/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/kernel/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -68,7 +68,7 @@ struct msm_pinctrl {
 
 	bool intr_target_use_scm;
 
-	raw_spinlock_t lock;
+	hard_spinlock_t lock;
 
 	DECLARE_BITMAP(dual_edge_irqs, MAX_NR_GPIO);
 	DECLARE_BITMAP(enabled_irqs, MAX_NR_GPIO);
@@ -1273,7 +1273,8 @@ static int msm_gpio_init(struct msm_pinctrl *pctrl)
 	pctrl->irq_chip.irq_set_vcpu_affinity = msm_gpio_irq_set_vcpu_affinity;
 	pctrl->irq_chip.flags = IRQCHIP_MASK_ON_SUSPEND |
 				IRQCHIP_SET_TYPE_MASKED |
-				IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND;
+				IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND |
+				IRQCHIP_PIPELINE_SAFE;
 
 	np = of_parse_phandle(pctrl->dev->of_node, "wakeup-parent", 0);
 	if (np) {
diff --git a/kernel/drivers/pinctrl/samsung/pinctrl-exynos.c b/kernel/drivers/pinctrl/samsung/pinctrl-exynos.c
index 493079a47d..7da7f80e65 100644
--- a/kernel/drivers/pinctrl/samsung/pinctrl-exynos.c
+++ b/kernel/drivers/pinctrl/samsung/pinctrl-exynos.c
@@ -58,13 +58,13 @@ static void exynos_irq_mask(struct irq_data *irqd)
 	unsigned int mask;
 	unsigned long flags;
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 
 	mask = readl(bank->eint_base + reg_mask);
 	mask |= 1 << irqd->hwirq;
 	writel(mask, bank->eint_base + reg_mask);
 
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 }
 
 static void exynos_irq_ack(struct irq_data *irqd)
@@ -97,13 +97,13 @@ static void exynos_irq_unmask(struct irq_data *irqd)
 	if (irqd_get_trigger_type(irqd) & IRQ_TYPE_LEVEL_MASK)
 		exynos_irq_ack(irqd);
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 
 	mask = readl(bank->eint_base + reg_mask);
 	mask &= ~(1 << irqd->hwirq);
 	writel(mask, bank->eint_base + reg_mask);
 
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 }
 
 static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type)
@@ -169,14 +169,14 @@ static int exynos_irq_request_resources(struct irq_data *irqd)
 	shift = irqd->hwirq * bank_type->fld_width[PINCFG_TYPE_FUNC];
 	mask = (1 << bank_type->fld_width[PINCFG_TYPE_FUNC]) - 1;
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 
 	con = readl(bank->pctl_base + reg_con);
 	con &= ~(mask << shift);
 	con |= EXYNOS_PIN_FUNC_EINT << shift;
 	writel(con, bank->pctl_base + reg_con);
 
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 
 	return 0;
 }
@@ -192,14 +192,14 @@ static void exynos_irq_release_resources(struct irq_data *irqd)
 	shift = irqd->hwirq * bank_type->fld_width[PINCFG_TYPE_FUNC];
 	mask = (1 << bank_type->fld_width[PINCFG_TYPE_FUNC]) - 1;
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 
 	con = readl(bank->pctl_base + reg_con);
 	con &= ~(mask << shift);
 	con |= EXYNOS_PIN_FUNC_INPUT << shift;
 	writel(con, bank->pctl_base + reg_con);
 
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 
 	gpiochip_unlock_as_irq(&bank->gpio_chip, irqd->hwirq);
 }
@@ -216,6 +216,7 @@ static const struct exynos_irq_chip exynos_gpio_irq_chip __initconst = {
 		.irq_set_type = exynos_irq_set_type,
 		.irq_request_resources = exynos_irq_request_resources,
 		.irq_release_resources = exynos_irq_release_resources,
+		.flags = IRQCHIP_PIPELINE_SAFE,
 	},
 	.eint_con = EXYNOS_GPIO_ECON_OFFSET,
 	.eint_mask = EXYNOS_GPIO_EMASK_OFFSET,
@@ -287,7 +288,7 @@ __init int exynos_eint_gpio_init(struct samsung_pinctrl_drv_data *d)
 	}
 
 	ret = devm_request_irq(dev, d->irq, exynos_eint_gpio_irq,
-					0, dev_name(dev), d);
+					IRQF_OOB, dev_name(dev), d);
 	if (ret) {
 		dev_err(dev, "irq request failed\n");
 		return -ENXIO;
@@ -305,6 +306,7 @@ __init int exynos_eint_gpio_init(struct samsung_pinctrl_drv_data *d)
 			goto err_domains;
 		}
 		bank->irq_chip->chip.name = bank->name;
+		bank->irq_chip->chip.flags |= IRQCHIP_PIPELINE_SAFE;
 
 		bank->irq_domain = irq_domain_add_linear(bank->of_node,
 				bank->nr_pins, &exynos_eint_irqd_ops, bank);
@@ -408,6 +410,7 @@ static const struct exynos_irq_chip s5pv210_wkup_irq_chip __initconst = {
 		.irq_set_wake = exynos_wkup_irq_set_wake,
 		.irq_request_resources = exynos_irq_request_resources,
 		.irq_release_resources = exynos_irq_release_resources,
+		.flags = IRQCHIP_PIPELINE_SAFE,
 	},
 	.eint_con = EXYNOS_WKUP_ECON_OFFSET,
 	.eint_mask = EXYNOS_WKUP_EMASK_OFFSET,
@@ -428,6 +431,7 @@ static const struct exynos_irq_chip exynos4210_wkup_irq_chip __initconst = {
 		.irq_set_wake = exynos_wkup_irq_set_wake,
 		.irq_request_resources = exynos_irq_request_resources,
 		.irq_release_resources = exynos_irq_release_resources,
+		.flags = IRQCHIP_PIPELINE_SAFE,
 	},
 	.eint_con = EXYNOS_WKUP_ECON_OFFSET,
 	.eint_mask = EXYNOS_WKUP_EMASK_OFFSET,
@@ -447,6 +451,7 @@ static const struct exynos_irq_chip exynos7_wkup_irq_chip __initconst = {
 		.irq_set_wake = exynos_wkup_irq_set_wake,
 		.irq_request_resources = exynos_irq_request_resources,
 		.irq_release_resources = exynos_irq_release_resources,
+		.flags = IRQCHIP_PIPELINE_SAFE,
 	},
 	.eint_con = EXYNOS7_WKUP_ECON_OFFSET,
 	.eint_mask = EXYNOS7_WKUP_EMASK_OFFSET,
diff --git a/kernel/drivers/pinctrl/samsung/pinctrl-samsung.c b/kernel/drivers/pinctrl/samsung/pinctrl-samsung.c
index 56fff83a14..f88f9f9b10 100644
--- a/kernel/drivers/pinctrl/samsung/pinctrl-samsung.c
+++ b/kernel/drivers/pinctrl/samsung/pinctrl-samsung.c
@@ -400,14 +400,14 @@ static void samsung_pinmux_setup(struct pinctrl_dev *pctldev, unsigned selector,
 		reg += 4;
 	}
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 
 	data = readl(reg + type->reg_offset[PINCFG_TYPE_FUNC]);
 	data &= ~(mask << shift);
 	data |= func->val << shift;
 	writel(data, reg + type->reg_offset[PINCFG_TYPE_FUNC]);
 
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 }
 
 /* enable a specified pinmux by writing to registers */
@@ -451,7 +451,7 @@ static int samsung_pinconf_rw(struct pinctrl_dev *pctldev, unsigned int pin,
 	width = type->fld_width[cfg_type];
 	cfg_reg = type->reg_offset[cfg_type];
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 
 	mask = (1 << width) - 1;
 	shift = pin_offset * width;
@@ -468,7 +468,7 @@ static int samsung_pinconf_rw(struct pinctrl_dev *pctldev, unsigned int pin,
 		*config = PINCFG_PACK(cfg_type, data);
 	}
 
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 
 	return 0;
 }
@@ -561,9 +561,9 @@ static void samsung_gpio_set(struct gpio_chip *gc, unsigned offset, int value)
 	struct samsung_pin_bank *bank = gpiochip_get_data(gc);
 	unsigned long flags;
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 	samsung_gpio_set_value(gc, offset, value);
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 }
 
 /* gpiolib gpio_get callback function */
@@ -626,9 +626,9 @@ static int samsung_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 	unsigned long flags;
 	int ret;
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 	ret = samsung_gpio_set_direction(gc, offset, true);
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 	return ret;
 }
 
@@ -640,10 +640,10 @@ static int samsung_gpio_direction_output(struct gpio_chip *gc, unsigned offset,
 	unsigned long flags;
 	int ret;
 
-	spin_lock_irqsave(&bank->slock, flags);
+	raw_spin_lock_irqsave(&bank->slock, flags);
 	samsung_gpio_set_value(gc, offset, value);
 	ret = samsung_gpio_set_direction(gc, offset, false);
-	spin_unlock_irqrestore(&bank->slock, flags);
+	raw_spin_unlock_irqrestore(&bank->slock, flags);
 
 	return ret;
 }
@@ -1067,7 +1067,7 @@ samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d,
 		bank->eint_offset = bdata->eint_offset;
 		bank->name = bdata->name;
 
-		spin_lock_init(&bank->slock);
+		raw_spin_lock_init(&bank->slock);
 		bank->drvdata = d;
 		bank->pin_base = d->nr_pins;
 		d->nr_pins += bank->nr_pins;
diff --git a/kernel/drivers/pinctrl/samsung/pinctrl-samsung.h b/kernel/drivers/pinctrl/samsung/pinctrl-samsung.h
index 379f34a9a4..59ce47aaaa 100644
--- a/kernel/drivers/pinctrl/samsung/pinctrl-samsung.h
+++ b/kernel/drivers/pinctrl/samsung/pinctrl-samsung.h
@@ -171,7 +171,7 @@ struct samsung_pin_bank {
 	struct gpio_chip gpio_chip;
 	struct pinctrl_gpio_range grange;
 	struct exynos_irq_chip *irq_chip;
-	spinlock_t slock;
+	hard_spinlock_t slock;
 
 	u32 pm_save[PINCFG_TYPE_NUM + 1]; /* +1 to handle double CON registers*/
 };
diff --git a/kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index e4b41cc6c5..a6170c0b86 100644
--- a/kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -1086,7 +1086,7 @@ static struct irq_chip sunxi_pinctrl_edge_irq_chip = {
 	.irq_release_resources = sunxi_pinctrl_irq_release_resources,
 	.irq_set_type	= sunxi_pinctrl_irq_set_type,
 	.irq_set_wake	= sunxi_pinctrl_irq_set_wake,
-	.flags		= IRQCHIP_MASK_ON_SUSPEND,
+	.flags		= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_PIPELINE_SAFE,
 };
 
 static struct irq_chip sunxi_pinctrl_level_irq_chip = {
@@ -1104,7 +1104,8 @@ static struct irq_chip sunxi_pinctrl_level_irq_chip = {
 	.irq_set_wake	= sunxi_pinctrl_irq_set_wake,
 	.flags		= IRQCHIP_EOI_THREADED |
 			  IRQCHIP_MASK_ON_SUSPEND |
-			  IRQCHIP_EOI_IF_HANDLED,
+			  IRQCHIP_EOI_IF_HANDLED |
+			  IRQCHIP_PIPELINE_SAFE,
 };
 
 static int sunxi_pinctrl_irq_of_xlate(struct irq_domain *d,
diff --git a/kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.h b/kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.h
index a32bb5bcb7..a1849aab80 100644
--- a/kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.h
+++ b/kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.h
@@ -167,7 +167,7 @@ struct sunxi_pinctrl {
 	unsigned			ngroups;
 	int				*irq;
 	unsigned			*irq_array;
-	raw_spinlock_t			lock;
+	hard_spinlock_t			lock;
 	struct pinctrl_dev		*pctl_dev;
 	unsigned long			variant;
 };
diff --git a/kernel/drivers/soc/qcom/smp2p.c b/kernel/drivers/soc/qcom/smp2p.c
index fb76c8bc3c..9eecb34ff0 100644
--- a/kernel/drivers/soc/qcom/smp2p.c
+++ b/kernel/drivers/soc/qcom/smp2p.c
@@ -281,6 +281,7 @@ static struct irq_chip smp2p_irq_chip = {
 	.irq_mask       = smp2p_mask_irq,
 	.irq_unmask     = smp2p_unmask_irq,
 	.irq_set_type	= smp2p_set_irq_type,
+	.flags		= IRQCHIP_PIPELINE_SAFE,
 };
 
 static int smp2p_irq_map(struct irq_domain *d,
diff --git a/kernel/drivers/soc/ti/ti_sci_inta_msi.c b/kernel/drivers/soc/ti/ti_sci_inta_msi.c
index 0eb9462f60..21d222b907 100644
--- a/kernel/drivers/soc/ti/ti_sci_inta_msi.c
+++ b/kernel/drivers/soc/ti/ti_sci_inta_msi.c
@@ -42,6 +42,7 @@ static void ti_sci_inta_msi_update_chip_ops(struct msi_domain_info *info)
 	chip->irq_unmask = irq_chip_unmask_parent;
 	chip->irq_mask = irq_chip_mask_parent;
 	chip->irq_ack = irq_chip_ack_parent;
+	chip->flags |= IRQCHIP_PIPELINE_SAFE;
 }
 
 struct irq_domain *ti_sci_inta_msi_create_irq_domain(struct fwnode_handle *fwnode,
diff --git a/kernel/drivers/spi/Kconfig b/kernel/drivers/spi/Kconfig
index 16b31f361c..6744f1e0bd 100755
--- a/kernel/drivers/spi/Kconfig
+++ b/kernel/drivers/spi/Kconfig
@@ -32,6 +32,10 @@ config SPI_DEBUG
 	  Say "yes" to enable debug messaging (like dev_dbg and pr_debug),
 	  sysfs, and debugfs support in SPI controller and protocol drivers.
 
+config SPI_OOB
+	def_bool n
+	depends on HAS_DMA && DOVETAIL
+
 #
 # MASTER side ... talking to discrete SPI slave chips including microcontrollers
 #
@@ -138,6 +142,13 @@ config SPI_BCM2835
 	  is for the regular SPI controller. Slave mode operation is not also
 	  not supported.
 
+config SPI_BCM2835_OOB
+	bool "Out-of-band support for BCM2835 SPI controller"
+	depends on SPI_BCM2835 && DOVETAIL
+	select SPI_OOB
+	help
+	  Enable out-of-band cyclic transfers.
+
 config SPI_BCM2835AUX
 	tristate "BCM2835 SPI auxiliary controller"
 	depends on ((ARCH_BCM2835 || ARCH_BRCMSTB) && GPIOLIB) || COMPILE_TEST
diff --git a/kernel/drivers/spi/spi-bcm2835.c b/kernel/drivers/spi/spi-bcm2835.c
index bb9d8386ba..4a3dbc0d7e 100644
--- a/kernel/drivers/spi/spi-bcm2835.c
+++ b/kernel/drivers/spi/spi-bcm2835.c
@@ -1079,17 +1079,10 @@ static int bcm2835_spi_transfer_one_poll(struct spi_controller *ctlr,
 	return 0;
 }
 
-static int bcm2835_spi_transfer_one(struct spi_controller *ctlr,
-				    struct spi_device *spi,
-				    struct spi_transfer *tfr)
+static unsigned long bcm2835_get_clkdiv(struct bcm2835_spi *bs, u32 spi_hz,
+					u32 *effective_speed_hz)
 {
-	struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
-	unsigned long spi_hz, cdiv;
-	unsigned long hz_per_byte, byte_limit;
-	u32 cs = bs->prepare_cs[spi->chip_select];
-
-	/* set clock */
-	spi_hz = tfr->speed_hz;
+	unsigned long cdiv;
 
 	if (spi_hz >= bs->clk_hz / 2) {
 		cdiv = 2; /* clk_hz/2 is the fastest we can go */
@@ -1103,7 +1096,25 @@ static int bcm2835_spi_transfer_one(struct spi_controller *ctlr,
 	} else {
 		cdiv = 0; /* 0 is the slowest we can go */
 	}
-	tfr->effective_speed_hz = cdiv ? (bs->clk_hz / cdiv) : (bs->clk_hz / 65536);
+
+	*effective_speed_hz = cdiv ? (bs->clk_hz / cdiv) : (bs->clk_hz / 65536);
+
+	return cdiv;
+}
+
+static int bcm2835_spi_transfer_one(struct spi_controller *ctlr,
+				    struct spi_device *spi,
+				    struct spi_transfer *tfr)
+{
+	struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
+	unsigned long spi_hz, cdiv;
+	unsigned long hz_per_byte, byte_limit;
+	u32 cs = bs->prepare_cs[spi->chip_select];
+
+	/* set clock */
+	spi_hz = tfr->speed_hz;
+
+	cdiv = bcm2835_get_clkdiv(bs, spi_hz, &tfr->effective_speed_hz);
 	bcm2835_wr(bs, BCM2835_SPI_CLK, cdiv);
 
 	/* handle all the 3-wire mode */
@@ -1283,6 +1294,68 @@ static int bcm2835_spi_setup(struct spi_device *spi)
 	return 0;
 }
 
+#ifdef CONFIG_SPI_BCM2835_OOB
+
+static int bcm2835_spi_prepare_oob_transfer(struct spi_controller *ctlr,
+					struct spi_oob_transfer *xfer)
+{
+	/*
+	 * The size of a transfer is limited by DLEN which is 16-bit
+	 * wide, and we don't want to scatter transfers in out-of-band
+	 * mode, so cap the frame size accordingly.
+	 */
+	if (xfer->setup.frame_len > 65532)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void bcm2835_spi_start_oob_transfer(struct spi_controller *ctlr,
+					struct spi_oob_transfer *xfer)
+{
+	struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
+	struct spi_device *spi = xfer->spi;
+	u32 cs = bs->prepare_cs[spi->chip_select], effective_speed_hz;
+	unsigned long cdiv;
+
+	/* See bcm2835_spi_prepare_message(). */
+	bcm2835_wr(bs, BCM2835_SPI_CS, cs);
+
+	cdiv = bcm2835_get_clkdiv(bs, xfer->setup.speed_hz, &effective_speed_hz);
+	xfer->effective_speed_hz = effective_speed_hz;
+	bcm2835_wr(bs, BCM2835_SPI_CLK, cdiv);
+	bcm2835_wr(bs, BCM2835_SPI_DLEN, xfer->setup.frame_len);
+
+	if (spi->mode & SPI_3WIRE)
+		cs |= BCM2835_SPI_CS_REN;
+	bcm2835_wr(bs, BCM2835_SPI_CS,
+		   cs | BCM2835_SPI_CS_TA | BCM2835_SPI_CS_DMAEN);
+}
+
+static void bcm2835_spi_pulse_oob_transfer(struct spi_controller *ctlr,
+					struct spi_oob_transfer *xfer)
+{
+	struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
+
+	/* Reload DLEN for the next pulse. */
+	bcm2835_wr(bs, BCM2835_SPI_DLEN, xfer->setup.frame_len);
+}
+
+static void bcm2835_spi_terminate_oob_transfer(struct spi_controller *ctlr,
+					struct spi_oob_transfer *xfer)
+{
+	struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
+
+	bcm2835_spi_reset_hw(bs);
+}
+
+#else
+#define bcm2835_spi_prepare_oob_transfer	NULL
+#define bcm2835_spi_start_oob_transfer		NULL
+#define bcm2835_spi_pulse_oob_transfer		NULL
+#define bcm2835_spi_terminate_oob_transfer	NULL
+#endif
+
 static int bcm2835_spi_probe(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr;
@@ -1304,6 +1377,10 @@ static int bcm2835_spi_probe(struct platform_device *pdev)
 	ctlr->transfer_one = bcm2835_spi_transfer_one;
 	ctlr->handle_err = bcm2835_spi_handle_err;
 	ctlr->prepare_message = bcm2835_spi_prepare_message;
+	ctlr->prepare_oob_transfer = bcm2835_spi_prepare_oob_transfer;
+	ctlr->start_oob_transfer = bcm2835_spi_start_oob_transfer;
+	ctlr->pulse_oob_transfer = bcm2835_spi_pulse_oob_transfer;
+	ctlr->terminate_oob_transfer = bcm2835_spi_terminate_oob_transfer;
 	ctlr->dev.of_node = pdev->dev.of_node;
 
 	bs = spi_controller_get_devdata(ctlr);
diff --git a/kernel/drivers/spi/spi.c b/kernel/drivers/spi/spi.c
index b1a638d91b..206e245387 100644
--- a/kernel/drivers/spi/spi.c
+++ b/kernel/drivers/spi/spi.c
@@ -2729,6 +2729,9 @@ int spi_register_controller(struct spi_controller *ctlr)
 	spin_lock_init(&ctlr->bus_lock_spinlock);
 	mutex_init(&ctlr->bus_lock_mutex);
 	mutex_init(&ctlr->io_mutex);
+#ifdef CONFIG_SPI_OOB
+	sema_init(&ctlr->bus_oob_lock_sem, 1);
+#endif
 	ctlr->bus_lock_flag = 0;
 	init_completion(&ctlr->xfer_completion);
 	if (!ctlr->max_dma_len)
@@ -3804,6 +3807,22 @@ EXPORT_SYMBOL_GPL(spi_async_locked);
  * inline functions.
  */
 
+static void get_spi_bus(struct spi_controller *ctlr)
+{
+	mutex_lock(&ctlr->bus_lock_mutex);
+#ifdef CONFIG_SPI_OOB
+	down(&ctlr->bus_oob_lock_sem);
+#endif
+}
+
+static void put_spi_bus(struct spi_controller *ctlr)
+{
+#ifdef CONFIG_SPI_OOB
+	up(&ctlr->bus_oob_lock_sem);
+#endif
+	mutex_unlock(&ctlr->bus_lock_mutex);
+}
+
 static void spi_complete(void *arg)
 {
 	complete(arg);
@@ -3888,9 +3907,9 @@ int spi_sync(struct spi_device *spi, struct spi_message *message)
 {
 	int ret;
 
-	mutex_lock(&spi->controller->bus_lock_mutex);
+	get_spi_bus(spi->controller);
 	ret = __spi_sync(spi, message);
-	mutex_unlock(&spi->controller->bus_lock_mutex);
+	put_spi_bus(spi->controller);
 
 	return ret;
 }
@@ -3937,7 +3956,7 @@ int spi_bus_lock(struct spi_controller *ctlr)
 {
 	unsigned long flags;
 
-	mutex_lock(&ctlr->bus_lock_mutex);
+	get_spi_bus(ctlr);
 
 	spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags);
 	ctlr->bus_lock_flag = 1;
@@ -3966,7 +3985,7 @@ int spi_bus_unlock(struct spi_controller *ctlr)
 {
 	ctlr->bus_lock_flag = 0;
 
-	mutex_unlock(&ctlr->bus_lock_mutex);
+	put_spi_bus(ctlr);
 
 	return 0;
 }
@@ -4051,6 +4070,274 @@ int spi_write_then_read(struct spi_device *spi,
 }
 EXPORT_SYMBOL_GPL(spi_write_then_read);
 
+#ifdef CONFIG_SPI_OOB
+
+static int bus_lock_oob(struct spi_controller *ctlr)
+{
+	unsigned long flags;
+	int ret = -EBUSY;
+
+	mutex_lock(&ctlr->bus_lock_mutex);
+
+	spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags);
+
+	if (!ctlr->bus_lock_flag && !down_trylock(&ctlr->bus_oob_lock_sem)) {
+		ctlr->bus_lock_flag = 1;
+		ret = 0;
+	}
+
+	spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags);
+
+	mutex_unlock(&ctlr->bus_lock_mutex);
+
+	return ret;
+}
+
+static int bus_unlock_oob(struct spi_controller *ctlr)
+{
+	ctlr->bus_lock_flag = 0;
+	up(&ctlr->bus_oob_lock_sem);
+
+	return 0;
+}
+
+static int prepare_oob_dma(struct spi_controller *ctlr,
+			struct spi_oob_transfer *xfer)
+{
+	struct dma_async_tx_descriptor *desc;
+	size_t len = xfer->setup.frame_len;
+	dma_cookie_t cookie;
+	dma_addr_t addr;
+	int ret;
+
+	/* TX to second half of I/O buffer. */
+	addr = xfer->dma_addr + xfer->aligned_frame_len;
+	desc = dmaengine_prep_slave_single(ctlr->dma_tx, addr, len,
+					DMA_MEM_TO_DEV,
+					DMA_OOB_INTERRUPT|DMA_OOB_PULSE);
+	if (!desc)
+		return -EIO;
+
+	xfer->txd = desc;
+	cookie = dmaengine_submit(desc);
+	ret = dma_submit_error(cookie);
+	if (ret)
+		return ret;
+
+	dma_async_issue_pending(ctlr->dma_tx);
+
+	/* RX to first half of I/O buffer. */
+	addr = xfer->dma_addr;
+	desc = dmaengine_prep_slave_single(ctlr->dma_rx, addr, len,
+					DMA_DEV_TO_MEM,
+					DMA_OOB_INTERRUPT|DMA_OOB_PULSE);
+	if (!desc) {
+		ret = -EIO;
+		goto fail_rx;
+	}
+
+	desc->callback = xfer->setup.xfer_done;
+	desc->callback_param = xfer;
+
+	xfer->rxd = desc;
+	cookie = dmaengine_submit(desc);
+	ret = dma_submit_error(cookie);
+	if (ret)
+		goto fail_rx;
+
+	dma_async_issue_pending(ctlr->dma_rx);
+
+	return 0;
+
+fail_rx:
+	dmaengine_terminate_sync(ctlr->dma_tx);
+
+	return ret;
+}
+
+static void unprepare_oob_dma(struct spi_controller *ctlr)
+{
+	dmaengine_terminate_sync(ctlr->dma_rx);
+	dmaengine_terminate_sync(ctlr->dma_tx);
+}
+
+/*
+ * A simpler version of __spi_validate() for oob transfers.
+ */
+static int validate_oob_xfer(struct spi_device *spi,
+			struct spi_oob_transfer *xfer)
+{
+	struct spi_controller *ctlr = spi->controller;
+	struct spi_oob_setup *p = &xfer->setup;
+	int w_size;
+
+	if (p->frame_len == 0)
+		return -EINVAL;
+
+	if (!p->bits_per_word)
+		p->bits_per_word = spi->bits_per_word;
+
+	if (!p->speed_hz)
+		p->speed_hz = spi->max_speed_hz;
+
+	if (ctlr->max_speed_hz && p->speed_hz > ctlr->max_speed_hz)
+		p->speed_hz = ctlr->max_speed_hz;
+
+	if (__spi_validate_bits_per_word(ctlr, p->bits_per_word))
+		return -EINVAL;
+
+	if (p->bits_per_word <= 8)
+		w_size = 1;
+	else if (p->bits_per_word <= 16)
+		w_size = 2;
+	else
+		w_size = 4;
+
+	if (p->frame_len % w_size)
+		return -EINVAL;
+
+	if (p->speed_hz && ctlr->min_speed_hz &&
+		p->speed_hz < ctlr->min_speed_hz)
+		return -EINVAL;
+
+	return 0;
+}
+
+int spi_prepare_oob_transfer(struct spi_device *spi,
+			struct spi_oob_transfer *xfer)
+{
+	struct spi_controller *ctlr;
+	dma_addr_t dma_addr;
+	size_t alen, iolen;
+	void *iobuf;
+	int ret;
+
+	/* Controller must support oob transactions. */
+	ctlr = spi->controller;
+	if (!ctlr->prepare_oob_transfer)
+		return -ENOTSUPP;
+
+	/* Out-of-band transfers require DMA support. */
+	if (!ctlr->can_dma)
+		return -ENODEV;
+
+	ret = validate_oob_xfer(spi, xfer);
+	if (ret)
+		return ret;
+
+	alen = L1_CACHE_ALIGN(xfer->setup.frame_len);
+	/*
+	 * Allocate a single coherent I/O buffer which is twice as
+	 * large as the user specified transfer length, TX data goes
+	 * to the upper half, RX data to the lower half.
+	 */
+	iolen = alen * 2;
+	iobuf = dma_alloc_coherent(ctlr->dev.parent, iolen,
+				&dma_addr, GFP_KERNEL);
+	if (iobuf == NULL)
+		return -ENOMEM;
+
+	xfer->spi = spi;
+	xfer->dma_addr = dma_addr;
+	xfer->io_buffer = iobuf;
+	xfer->aligned_frame_len = alen;
+	xfer->effective_speed_hz = 0;
+
+	ret = prepare_oob_dma(ctlr, xfer);
+	if (ret)
+		goto fail_prep_dma;
+
+	ret = bus_lock_oob(ctlr);
+	if (ret)
+		goto fail_bus_lock;
+
+	ret = ctlr->prepare_oob_transfer(ctlr, xfer);
+	if (ret)
+		goto fail_prep_xfer;
+
+	return 0;
+
+fail_prep_xfer:
+	bus_unlock_oob(ctlr);
+fail_bus_lock:
+	unprepare_oob_dma(ctlr);
+fail_prep_dma:
+	dma_free_coherent(ctlr->dev.parent, iolen, iobuf, dma_addr);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(spi_prepare_oob_transfer);
+
+void spi_start_oob_transfer(struct spi_oob_transfer *xfer)
+{
+	struct spi_device *spi = xfer->spi;
+	struct spi_controller *ctlr = spi->controller;
+
+	ctlr->start_oob_transfer(ctlr, xfer);
+}
+EXPORT_SYMBOL_GPL(spi_start_oob_transfer);
+
+int spi_pulse_oob_transfer(struct spi_oob_transfer *xfer) /* oob stage */
+{
+	struct spi_device *spi = xfer->spi;
+	struct spi_controller *ctlr = spi->controller;
+	int ret;
+
+	if (ctlr->pulse_oob_transfer)
+		ctlr->pulse_oob_transfer(ctlr, xfer);
+
+	ret = dma_pulse_oob(ctlr->dma_rx);
+	if (likely(!ret))
+		ret = dma_pulse_oob(ctlr->dma_tx);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(spi_pulse_oob_transfer);
+
+void spi_terminate_oob_transfer(struct spi_oob_transfer *xfer)
+{
+	struct spi_device *spi = xfer->spi;
+	struct spi_controller *ctlr = spi->controller;
+
+	if (ctlr->terminate_oob_transfer)
+		ctlr->terminate_oob_transfer(ctlr, xfer);
+
+	unprepare_oob_dma(ctlr);
+	bus_unlock_oob(ctlr);
+	dma_free_coherent(ctlr->dev.parent, xfer->aligned_frame_len * 2,
+			xfer->io_buffer, xfer->dma_addr);
+}
+EXPORT_SYMBOL_GPL(spi_terminate_oob_transfer);
+
+int spi_mmap_oob_transfer(struct vm_area_struct *vma,
+			struct spi_oob_transfer *xfer)
+{
+	struct spi_device *spi = xfer->spi;
+	struct spi_controller *ctlr = spi->controller;
+	size_t len;
+	int ret;
+
+	/*
+	 * We may have an IOMMU, rely on dma_mmap_coherent() for
+	 * dealing with the nitty-gritty details of mapping a coherent
+	 * buffer.
+	 */
+	len = vma->vm_end - vma->vm_start;
+	if (spi_get_oob_iolen(xfer) <= len)
+		ret = dma_mmap_coherent(ctlr->dev.parent,
+					vma,
+					xfer->io_buffer,
+					xfer->dma_addr,
+					len);
+	else
+		ret = -EINVAL;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(spi_mmap_oob_transfer);
+
+#endif	/* SPI_OOB */
+
 /*-------------------------------------------------------------------------*/
 
 #if IS_ENABLED(CONFIG_OF)
diff --git a/kernel/drivers/spmi/spmi-pmic-arb.c b/kernel/drivers/spmi/spmi-pmic-arb.c
index e6de2aeece..10669347b5 100644
--- a/kernel/drivers/spmi/spmi-pmic-arb.c
+++ b/kernel/drivers/spmi/spmi-pmic-arb.c
@@ -145,7 +145,7 @@ struct spmi_pmic_arb {
 	void __iomem		*cnfg;
 	void __iomem		*core;
 	resource_size_t		core_size;
-	raw_spinlock_t		lock;
+	hard_spinlock_t		lock;
 	u8			channel;
 	int			irq;
 	u8			ee;
@@ -684,7 +684,7 @@ static struct irq_chip pmic_arb_irqchip = {
 	.irq_set_type	= qpnpint_irq_set_type,
 	.irq_set_wake	= qpnpint_irq_set_wake,
 	.irq_get_irqchip_state	= qpnpint_get_irqchip_state,
-	.flags		= IRQCHIP_MASK_ON_SUSPEND,
+	.flags		= IRQCHIP_MASK_ON_SUSPEND|IRQCHIP_PIPELINE_SAFE,
 };
 
 static int qpnpint_irq_domain_translate(struct irq_domain *d,
diff --git a/kernel/drivers/tty/serial/8250/8250_core.c b/kernel/drivers/tty/serial/8250/8250_core.c
index 00f6dc7e94..da7ba88861 100644
--- a/kernel/drivers/tty/serial/8250/8250_core.c
+++ b/kernel/drivers/tty/serial/8250/8250_core.c
@@ -675,6 +675,48 @@ static int univ8250_console_match(struct console *co, char *name, int idx,
 	return -ENODEV;
 }
 
+#ifdef CONFIG_RAW_PRINTK
+
+static void raw_write_char(struct uart_8250_port *up, int c)
+{
+	unsigned int status, tmout = 10000;
+
+	for (;;) {
+		status = serial_in(up, UART_LSR);
+		up->lsr_saved_flags |= status & LSR_SAVE_FLAGS;
+		if ((status & UART_LSR_THRE) == UART_LSR_THRE)
+			break;
+		if (--tmout == 0)
+			break;
+		cpu_relax();
+	}
+	serial_port_out(&up->port, UART_TX, c);
+}
+
+static void univ8250_console_write_raw(struct console *co, const char *s,
+				       unsigned int count)
+{
+	struct uart_8250_port *up = &serial8250_ports[co->index];
+	unsigned int ier;
+
+        ier = serial_in(up, UART_IER);
+
+        if (up->capabilities & UART_CAP_UUE)
+                serial_out(up, UART_IER, UART_IER_UUE);
+        else
+                serial_out(up, UART_IER, 0);
+
+	while (count-- > 0) {
+		if (*s == '\n')
+			raw_write_char(up, '\r');
+		raw_write_char(up, *s++);
+	}
+
+        serial_out(up, UART_IER, ier);
+}
+
+#endif
+
 static struct console univ8250_console = {
 	.name		= "ttyS",
 	.write		= univ8250_console_write,
@@ -682,6 +724,9 @@ static struct console univ8250_console = {
 	.setup		= univ8250_console_setup,
 	.exit		= univ8250_console_exit,
 	.match		= univ8250_console_match,
+#ifdef CONFIG_RAW_PRINTK
+	.write_raw	= univ8250_console_write_raw,
+#endif
 	.flags		= CON_PRINTBUFFER | CON_ANYTIME,
 	.index		= -1,
 	.data		= &serial8250_reg,
diff --git a/kernel/drivers/tty/serial/amba-pl011.c b/kernel/drivers/tty/serial/amba-pl011.c
index 9900ee3f90..6ea53e1cc7 100644
--- a/kernel/drivers/tty/serial/amba-pl011.c
+++ b/kernel/drivers/tty/serial/amba-pl011.c
@@ -1887,6 +1887,8 @@ static void pl011_shutdown(struct uart_port *port)
 
 	pl011_disable_uart(uap);
 
+	if (IS_ENABLED(CONFIG_RAW_PRINTK))
+		clk_disable(uap->clk);
 	/*
 	 * Shut down the clock producer
 	 */
@@ -2194,6 +2196,37 @@ static void pl011_console_putchar(struct uart_port *port, int ch)
 	pl011_write(ch, uap, REG_DR);
 }
 
+#ifdef CONFIG_RAW_PRINTK
+
+static void
+pl011_console_write_raw(struct console *co, const char *s, unsigned int count)
+{
+	struct uart_amba_port *uap = amba_ports[co->index];
+	unsigned int old_cr = 0, new_cr;
+
+	if (!uap->vendor->always_enabled) {
+		old_cr = pl011_read(uap, REG_CR);
+		new_cr = old_cr & ~UART011_CR_CTSEN;
+		new_cr |= UART01x_CR_UARTEN | UART011_CR_TXE;
+		pl011_write(new_cr, uap, REG_CR);
+	}
+
+	while (count-- > 0) {
+		if (*s == '\n')
+			pl011_console_putchar(&uap->port, '\r');
+		pl011_console_putchar(&uap->port, *s++);
+	}
+
+	while ((pl011_read(uap, REG_FR) ^ uap->vendor->inv_fr)
+		& uap->vendor->fr_busy)
+		cpu_relax();
+
+	if (!uap->vendor->always_enabled)
+		pl011_write(old_cr, uap, REG_CR);
+}
+
+#endif  /* !CONFIG_RAW_PRINTK */
+
 static void
 pl011_console_write(struct console *co, const char *s, unsigned int count)
 {
@@ -2323,6 +2356,9 @@ static int pl011_console_setup(struct console *co, char *options)
 			pl011_console_get_options(uap, &baud, &parity, &bits);
 	}
 
+	if (IS_ENABLED(CONFIG_RAW_PRINTK))
+		clk_enable(uap->clk);
+
 	return uart_set_options(&uap->port, co, baud, parity, bits, flow);
 }
 
@@ -2393,6 +2429,9 @@ static struct console amba_console = {
 	.device		= uart_console_device,
 	.setup		= pl011_console_setup,
 	.match		= pl011_console_match,
+#ifdef CONFIG_RAW_PRINTK
+	.write_raw	= pl011_console_write_raw,
+#endif
 	.flags		= CON_PRINTBUFFER | CON_ANYTIME,
 	.index		= -1,
 	.data		= &amba_reg,
diff --git a/kernel/drivers/tty/serial/imx.c b/kernel/drivers/tty/serial/imx.c
index 164597e2e0..fa7d220011 100644
--- a/kernel/drivers/tty/serial/imx.c
+++ b/kernel/drivers/tty/serial/imx.c
@@ -1998,24 +1998,11 @@ static void imx_uart_console_putchar(struct uart_port *port, int ch)
 	imx_uart_writel(sport, ch, URTX0);
 }
 
-/*
- * Interrupts are disabled on entering
- */
 static void
-imx_uart_console_write(struct console *co, const char *s, unsigned int count)
+__imx_uart_console_write(struct imx_port *sport, const char *s, unsigned int count)
 {
-	struct imx_port *sport = imx_uart_ports[co->index];
 	struct imx_port_ucrs old_ucr;
 	unsigned int ucr1;
-	unsigned long flags = 0;
-	int locked = 1;
-
-	if (sport->port.sysrq)
-		locked = 0;
-	else if (oops_in_progress)
-		locked = spin_trylock_irqsave(&sport->port.lock, flags);
-	else
-		spin_lock_irqsave(&sport->port.lock, flags);
 
 	/*
 	 *	First, save UCR1/2/3 and then disable interrupts
@@ -2041,11 +2028,41 @@ imx_uart_console_write(struct console *co, const char *s, unsigned int count)
 	while (!(imx_uart_readl(sport, USR2) & USR2_TXDC));
 
 	imx_uart_ucrs_restore(sport, &old_ucr);
+}
+
+/*
+ * Interrupts are disabled on entering
+ */
+static void
+imx_uart_console_write(struct console *co, const char *s, unsigned int count)
+{
+	struct imx_port *sport = imx_uart_ports[co->index];
+	unsigned long flags;
+	int locked = 1;
+
+	if (sport->port.sysrq)
+		locked = 0;
+	else if (oops_in_progress)
+		locked = spin_trylock_irqsave(&sport->port.lock, flags);
+	else
+		spin_lock_irqsave(&sport->port.lock, flags);
+
+	__imx_uart_console_write(sport, s, count);
 
 	if (locked)
 		spin_unlock_irqrestore(&sport->port.lock, flags);
 }
 
+#ifdef CONFIG_RAW_PRINTK
+static void
+imx_uart_console_write_raw(struct console *co, const char *s, unsigned int count)
+{
+	struct imx_port *sport = imx_uart_ports[co->index];
+
+	return __imx_uart_console_write(sport, s, count);
+}
+#endif
+
 /*
  * If the port was already initialised (eg, by a boot loader),
  * try to determine the current setup.
@@ -2161,6 +2178,9 @@ static struct uart_driver imx_uart_uart_driver;
 static struct console imx_uart_console = {
 	.name		= DEV_NAME,
 	.write		= imx_uart_console_write,
+#ifdef CONFIG_RAW_PRINTK
+	.write_raw	= imx_uart_console_write_raw,
+#endif
 	.device		= uart_console_device,
 	.setup		= imx_uart_console_setup,
 	.flags		= CON_PRINTBUFFER,
diff --git a/kernel/drivers/tty/serial/samsung_tty.c b/kernel/drivers/tty/serial/samsung_tty.c
index 263c33260d..a36d1f404f 100644
--- a/kernel/drivers/tty/serial/samsung_tty.c
+++ b/kernel/drivers/tty/serial/samsung_tty.c
@@ -2367,6 +2367,10 @@ static struct console s3c24xx_serial_console = {
 	.flags		= CON_PRINTBUFFER,
 	.index		= -1,
 	.write		= s3c24xx_serial_console_write,
+#ifdef CONFIG_RAW_PRINTK
+	/* The common write handler can run from atomic context. */
+	.write_raw	= s3c24xx_serial_console_write,
+#endif
 	.setup		= s3c24xx_serial_console_setup,
 	.data		= &s3c24xx_uart_drv,
 };
diff --git a/kernel/drivers/tty/serial/st-asc.c b/kernel/drivers/tty/serial/st-asc.c
index 97d36f870f..1239fd59be 100644
--- a/kernel/drivers/tty/serial/st-asc.c
+++ b/kernel/drivers/tty/serial/st-asc.c
@@ -908,6 +908,29 @@ static void asc_console_write(struct console *co, const char *s, unsigned count)
 		spin_unlock_irqrestore(&port->lock, flags);
 }
 
+#ifdef CONFIG_RAW_PRINTK
+
+static void asc_console_write_raw(struct console *co,
+				  const char *s, unsigned int count)
+{
+	struct uart_port *port = &asc_ports[co->index].port;
+	unsigned long timeout = 1000000;
+	u32 intenable;
+
+	intenable = asc_in(port, ASC_INTEN);
+	asc_out(port, ASC_INTEN, 0);
+	(void)asc_in(port, ASC_INTEN);	/* Defeat bus write posting */
+
+	uart_console_write(port, s, count, asc_console_putchar);
+
+	while (timeout-- && !asc_txfifo_is_empty(port))
+		cpu_relax();	/* wait shorter */
+
+	asc_out(port, ASC_INTEN, intenable);
+}
+
+#endif
+
 static int asc_console_setup(struct console *co, char *options)
 {
 	struct asc_port *ascport;
@@ -940,6 +963,9 @@ static struct console asc_console = {
 	.name		= ASC_SERIAL_NAME,
 	.device		= uart_console_device,
 	.write		= asc_console_write,
+#ifdef CONFIG_RAW_PRINTK
+	.write_raw	= asc_console_write_raw,
+#endif
 	.setup		= asc_console_setup,
 	.flags		= CON_PRINTBUFFER,
 	.index		= -1,
diff --git a/kernel/fs/eventfd.c b/kernel/fs/eventfd.c
index 4a14295cff..cb99868391 100644
--- a/kernel/fs/eventfd.c
+++ b/kernel/fs/eventfd.c
@@ -266,17 +266,17 @@ static ssize_t eventfd_read(struct kiocb *iocb, struct iov_iter *to)
 	return sizeof(ucnt);
 }
 
-static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t count,
-			     loff_t *ppos)
+static ssize_t eventfd_write(struct kiocb *iocb, struct iov_iter *from)
 {
+	struct file *file = iocb->ki_filp;
 	struct eventfd_ctx *ctx = file->private_data;
 	ssize_t res;
 	__u64 ucnt;
 	DECLARE_WAITQUEUE(wait, current);
 
-	if (count < sizeof(ucnt))
+	if (iov_iter_count(from) < sizeof(ucnt))
 		return -EINVAL;
-	if (copy_from_user(&ucnt, buf, sizeof(ucnt)))
+	if (copy_from_iter(&ucnt, sizeof(ucnt), from) != sizeof(ucnt))
 		return -EFAULT;
 	if (ucnt == ULLONG_MAX)
 		return -EINVAL;
@@ -333,7 +333,7 @@ static const struct file_operations eventfd_fops = {
 	.release	= eventfd_release,
 	.poll		= eventfd_poll,
 	.read_iter	= eventfd_read,
-	.write		= eventfd_write,
+	.write_iter	= eventfd_write,
 	.llseek		= noop_llseek,
 };
 
diff --git a/kernel/fs/exec.c b/kernel/fs/exec.c
index b798885e30..30f75c0bff 100644
--- a/kernel/fs/exec.c
+++ b/kernel/fs/exec.c
@@ -34,6 +34,7 @@
 #include <linux/swap.h>
 #include <linux/string.h>
 #include <linux/init.h>
+#include <linux/irq_pipeline.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/coredump.h>
 #include <linux/sched/signal.h>
@@ -68,6 +69,7 @@
 #include <linux/uaccess.h>
 #include <asm/mmu_context.h>
 #include <asm/tlb.h>
+#include <asm/dovetail.h>
 
 #include <trace/events/task.h>
 #include "internal.h"
@@ -981,6 +983,7 @@ static int exec_mmap(struct mm_struct *mm)
 	struct task_struct *tsk;
 	struct mm_struct *old_mm, *active_mm;
 	int ret;
+	unsigned long flags;
 
 	/* Notify parent that we're no longer interested in the old VM */
 	tsk = current;
@@ -1013,6 +1016,7 @@ static int exec_mmap(struct mm_struct *mm)
 
 	local_irq_disable();
 	active_mm = tsk->active_mm;
+	protect_inband_mm(flags);
 	tsk->active_mm = mm;
 	tsk->mm = mm;
 	/*
@@ -1021,10 +1025,17 @@ static int exec_mmap(struct mm_struct *mm)
 	 * lazy tlb mm refcounting when these are updated by context
 	 * switches. Not all architectures can handle irqs off over
 	 * activate_mm yet.
+	 *
+	 * irq_pipeline: activate_mm() allowing irqs off context is a
+	 * requirement. e.g. TLB shootdown must not involve IPIs. We
+	 * make sure protect_inband_mm() is in effect while switching
+	 * in and activating the new mm by forcing
+	 * CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM on.
 	 */
 	if (!IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
 		local_irq_enable();
 	activate_mm(active_mm, mm);
+	unprotect_inband_mm(flags);
 	if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM))
 		local_irq_enable();
 	tsk->mm->vmacache_seqnum = 0;
@@ -1302,6 +1313,9 @@ int begin_new_exec(struct linux_binprm * bprm)
 	if (retval)
 		goto out_unlock;
 
+	/* Tell Dovetail about the ongoing exec(). */
+	arch_dovetail_exec_prepare();
+
 	/*
 	 * Ensure that the uaccess routines can actually operate on userspace
 	 * pointers:
diff --git a/kernel/fs/fcntl.c b/kernel/fs/fcntl.c
index fcf34f83bf..0b131a8113 100644
--- a/kernel/fs/fcntl.c
+++ b/kernel/fs/fcntl.c
@@ -1044,7 +1044,7 @@ static int __init fcntl_init(void)
 	 * Exceptions: O_NONBLOCK is a two bit define on parisc; O_NDELAY
 	 * is defined as O_NONBLOCK on some platforms and not on others.
 	 */
-	BUILD_BUG_ON(21 - 1 /* for O_RDONLY being 0 */ !=
+	BUILD_BUG_ON(22 - 1 /* for O_RDONLY being 0 */ !=
 		HWEIGHT32(
 			(VALID_OPEN_FLAGS & ~(O_NONBLOCK | O_NDELAY)) |
 			__FMODE_EXEC | __FMODE_NONOTIFY));
diff --git a/kernel/fs/file.c b/kernel/fs/file.c
index 97a0cd31fa..6d84335392 100644
--- a/kernel/fs/file.c
+++ b/kernel/fs/file.c
@@ -429,6 +429,7 @@ static struct fdtable *close_files(struct files_struct * files)
 			if (set & 1) {
 				struct file * file = xchg(&fdt->fd[i], NULL);
 				if (file) {
+					uninstall_inband_fd(i, file, files);
 					filp_close(file, files);
 					cond_resched();
 				}
@@ -646,6 +647,7 @@ void __fd_install(struct files_struct *files, unsigned int fd,
 		fdt = files_fdtable(files);
 		BUG_ON(fdt->fd[fd] != NULL);
 		rcu_assign_pointer(fdt->fd[fd], file);
+		install_inband_fd(fd, file, files);
 		spin_unlock(&files->file_lock);
 		return;
 	}
@@ -654,6 +656,7 @@ void __fd_install(struct files_struct *files, unsigned int fd,
 	fdt = rcu_dereference_sched(files->fdt);
 	BUG_ON(fdt->fd[fd] != NULL);
 	rcu_assign_pointer(fdt->fd[fd], file);
+	install_inband_fd(fd, file, files);
 	rcu_read_unlock_sched();
 }
 
@@ -682,6 +685,7 @@ static struct file *pick_file(struct files_struct *files, unsigned fd)
 		goto out_unlock;
 	rcu_assign_pointer(fdt->fd[fd], NULL);
 	__put_unused_fd(files, fd);
+	uninstall_inband_fd(fd, file, files);
 
 out_unlock:
 	spin_unlock(&files->file_lock);
@@ -799,6 +803,8 @@ int __close_fd_get_file(unsigned int fd, struct file **res)
 		goto out_err;
 	rcu_assign_pointer(fdt->fd[fd], NULL);
 	__put_unused_fd(files, fd);
+	uninstall_inband_fd(fd, file, files);
+	spin_unlock(&files->file_lock);
 	get_file(file);
 	*res = file;
 	return 0;
@@ -850,6 +856,7 @@ void do_close_on_exec(struct files_struct *files)
 				continue;
 			rcu_assign_pointer(fdt->fd[fd], NULL);
 			__put_unused_fd(files, fd);
+			uninstall_inband_fd(fd, file, files);
 			spin_unlock(&files->file_lock);
 			filp_close(file, files);
 			cond_resched();
@@ -1088,6 +1095,7 @@ __releases(&files->file_lock)
 		__set_close_on_exec(fd, fdt);
 	else
 		__clear_close_on_exec(fd, fdt);
+	replace_inband_fd(fd, file, files);
 	spin_unlock(&files->file_lock);
 
 	if (tofree)
diff --git a/kernel/fs/ioctl.c b/kernel/fs/ioctl.c
index ac3b386d99..ad1d738359 100644
--- a/kernel/fs/ioctl.c
+++ b/kernel/fs/ioctl.c
@@ -790,6 +790,22 @@ long compat_ptr_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 }
 EXPORT_SYMBOL(compat_ptr_ioctl);
 
+/**
+ * compat_ptr_oob_ioctl - generic implementation of .compat_oob_ioctl file operation
+ *
+ * The equivalent of compat_ptr_ioctl, dealing with out-of-band ioctl
+ * calls. Management of this handler is delegated to the code
+ * implementing the out-of-band ioctl() syscall in the companion core.
+ */
+long compat_ptr_oob_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	if (!file->f_op->oob_ioctl)
+		return -ENOIOCTLCMD;
+
+	return file->f_op->oob_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
+}
+EXPORT_SYMBOL(compat_ptr_oob_ioctl);
+
 COMPAT_SYSCALL_DEFINE3(ioctl, unsigned int, fd, unsigned int, cmd,
 		       compat_ulong_t, arg)
 {
diff --git a/kernel/fs/udf/inode.c b/kernel/fs/udf/inode.c
index d32b836f6c..e94a18bb7f 100644
--- a/kernel/fs/udf/inode.c
+++ b/kernel/fs/udf/inode.c
@@ -438,6 +438,12 @@ static int udf_get_block(struct inode *inode, sector_t block,
 		iinfo->i_next_alloc_goal++;
 	}
 
+	/*
+	 * Block beyond EOF and prealloc extents? Just discard preallocation
+	 * as it is not useful and complicates things.
+	 */
+	if (((loff_t)block) << inode->i_blkbits > iinfo->i_lenExtents)
+		udf_discard_prealloc(inode);
 	udf_clear_extent_cache(inode);
 	phys = inode_getblk(inode, block, &err, &new);
 	if (!phys)
@@ -487,8 +493,6 @@ static int udf_do_extend_file(struct inode *inode,
 	uint32_t add;
 	int count = 0, fake = !(last_ext->extLength & UDF_EXTENT_LENGTH_MASK);
 	struct super_block *sb = inode->i_sb;
-	struct kernel_lb_addr prealloc_loc = {};
-	uint32_t prealloc_len = 0;
 	struct udf_inode_info *iinfo;
 	int err;
 
@@ -509,19 +513,6 @@ static int udf_do_extend_file(struct inode *inode,
 			~(sb->s_blocksize - 1);
 	}
 
-	/* Last extent are just preallocated blocks? */
-	if ((last_ext->extLength & UDF_EXTENT_FLAG_MASK) ==
-						EXT_NOT_RECORDED_ALLOCATED) {
-		/* Save the extent so that we can reattach it to the end */
-		prealloc_loc = last_ext->extLocation;
-		prealloc_len = last_ext->extLength;
-		/* Mark the extent as a hole */
-		last_ext->extLength = EXT_NOT_RECORDED_NOT_ALLOCATED |
-			(last_ext->extLength & UDF_EXTENT_LENGTH_MASK);
-		last_ext->extLocation.logicalBlockNum = 0;
-		last_ext->extLocation.partitionReferenceNum = 0;
-	}
-
 	/* Can we merge with the previous extent? */
 	if ((last_ext->extLength & UDF_EXTENT_FLAG_MASK) ==
 					EXT_NOT_RECORDED_NOT_ALLOCATED) {
@@ -549,7 +540,7 @@ static int udf_do_extend_file(struct inode *inode,
 		 * more extents, we may need to enter possible following
 		 * empty indirect extent.
 		 */
-		if (new_block_bytes || prealloc_len)
+		if (new_block_bytes)
 			udf_next_aext(inode, last_pos, &tmploc, &tmplen, 0);
 	}
 
@@ -583,17 +574,6 @@ static int udf_do_extend_file(struct inode *inode,
 	}
 
 out:
-	/* Do we have some preallocated blocks saved? */
-	if (prealloc_len) {
-		err = udf_add_aext(inode, last_pos, &prealloc_loc,
-				   prealloc_len, 1);
-		if (err)
-			return err;
-		last_ext->extLocation = prealloc_loc;
-		last_ext->extLength = prealloc_len;
-		count++;
-	}
-
 	/* last_pos should point to the last written extent... */
 	if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT)
 		last_pos->offset -= sizeof(struct short_ad);
@@ -609,13 +589,17 @@ static int udf_do_extend_file(struct inode *inode,
 static void udf_do_extend_final_block(struct inode *inode,
 				      struct extent_position *last_pos,
 				      struct kernel_long_ad *last_ext,
-				      uint32_t final_block_len)
+				      uint32_t new_elen)
 {
-	struct super_block *sb = inode->i_sb;
 	uint32_t added_bytes;
 
-	added_bytes = final_block_len -
-		      (last_ext->extLength & (sb->s_blocksize - 1));
+	/*
+	 * Extent already large enough? It may be already rounded up to block
+	 * size...
+	 */
+	if (new_elen <= (last_ext->extLength & UDF_EXTENT_LENGTH_MASK))
+		return;
+	added_bytes = (last_ext->extLength & UDF_EXTENT_LENGTH_MASK) - new_elen;
 	last_ext->extLength += added_bytes;
 	UDF_I(inode)->i_lenExtents += added_bytes;
 
@@ -632,12 +616,12 @@ static int udf_extend_file(struct inode *inode, loff_t newsize)
 	int8_t etype;
 	struct super_block *sb = inode->i_sb;
 	sector_t first_block = newsize >> sb->s_blocksize_bits, offset;
-	unsigned long partial_final_block;
+	loff_t new_elen;
 	int adsize;
 	struct udf_inode_info *iinfo = UDF_I(inode);
 	struct kernel_long_ad extent;
 	int err = 0;
-	int within_final_block;
+	bool within_last_ext;
 
 	if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT)
 		adsize = sizeof(struct short_ad);
@@ -646,8 +630,17 @@ static int udf_extend_file(struct inode *inode, loff_t newsize)
 	else
 		BUG();
 
+	/*
+	 * When creating hole in file, just don't bother with preserving
+	 * preallocation. It likely won't be very useful anyway.
+	 */
+	udf_discard_prealloc(inode);
+
 	etype = inode_bmap(inode, first_block, &epos, &eloc, &elen, &offset);
-	within_final_block = (etype != -1);
+	within_last_ext = (etype != -1);
+	/* We don't expect extents past EOF... */
+	WARN_ON_ONCE(within_last_ext &&
+		     elen > ((loff_t)offset + 1) << inode->i_blkbits);
 
 	if ((!epos.bh && epos.offset == udf_file_entry_alloc_offset(inode)) ||
 	    (epos.bh && epos.offset == sizeof(struct allocExtDesc))) {
@@ -663,19 +656,17 @@ static int udf_extend_file(struct inode *inode, loff_t newsize)
 		extent.extLength |= etype << 30;
 	}
 
-	partial_final_block = newsize & (sb->s_blocksize - 1);
+	new_elen = ((loff_t)offset << inode->i_blkbits) |
+					(newsize & (sb->s_blocksize - 1));
 
 	/* File has extent covering the new size (could happen when extending
 	 * inside a block)?
 	 */
-	if (within_final_block) {
+	if (within_last_ext) {
 		/* Extending file within the last file block */
-		udf_do_extend_final_block(inode, &epos, &extent,
-					  partial_final_block);
+		udf_do_extend_final_block(inode, &epos, &extent, new_elen);
 	} else {
-		loff_t add = ((loff_t)offset << sb->s_blocksize_bits) |
-			     partial_final_block;
-		err = udf_do_extend_file(inode, &epos, &extent, add);
+		err = udf_do_extend_file(inode, &epos, &extent, new_elen);
 	}
 
 	if (err < 0)
@@ -776,10 +767,11 @@ static sector_t inode_getblk(struct inode *inode, sector_t block,
 		goto out_free;
 	}
 
-	/* Are we beyond EOF? */
+	/* Are we beyond EOF and preallocated extent? */
 	if (etype == -1) {
 		int ret;
 		loff_t hole_len;
+
 		isBeyondEOF = true;
 		if (count) {
 			if (c)
diff --git a/kernel/fs/udf/truncate.c b/kernel/fs/udf/truncate.c
index 532cda9964..036ebd892b 100644
--- a/kernel/fs/udf/truncate.c
+++ b/kernel/fs/udf/truncate.c
@@ -120,60 +120,42 @@ void udf_truncate_tail_extent(struct inode *inode)
 
 void udf_discard_prealloc(struct inode *inode)
 {
-	struct extent_position epos = { NULL, 0, {0, 0} };
+	struct extent_position epos = {};
+	struct extent_position prev_epos = {};
 	struct kernel_lb_addr eloc;
 	uint32_t elen;
 	uint64_t lbcount = 0;
 	int8_t etype = -1, netype;
-	int adsize;
 	struct udf_inode_info *iinfo = UDF_I(inode);
+	int bsize = 1 << inode->i_blkbits;
 
 	if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB ||
-	    inode->i_size == iinfo->i_lenExtents)
+	    ALIGN(inode->i_size, bsize) == ALIGN(iinfo->i_lenExtents, bsize))
 		return;
 
-	if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT)
-		adsize = sizeof(struct short_ad);
-	else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG)
-		adsize = sizeof(struct long_ad);
-	else
-		adsize = 0;
-
 	epos.block = iinfo->i_location;
 
 	/* Find the last extent in the file */
-	while ((netype = udf_next_aext(inode, &epos, &eloc, &elen, 1)) != -1) {
-		etype = netype;
+	while ((netype = udf_next_aext(inode, &epos, &eloc, &elen, 0)) != -1) {
+		brelse(prev_epos.bh);
+		prev_epos = epos;
+		if (prev_epos.bh)
+			get_bh(prev_epos.bh);
+
+		etype = udf_next_aext(inode, &epos, &eloc, &elen, 1);
 		lbcount += elen;
 	}
 	if (etype == (EXT_NOT_RECORDED_ALLOCATED >> 30)) {
-		epos.offset -= adsize;
 		lbcount -= elen;
-		extent_trunc(inode, &epos, &eloc, etype, elen, 0);
-		if (!epos.bh) {
-			iinfo->i_lenAlloc =
-				epos.offset -
-				udf_file_entry_alloc_offset(inode);
-			mark_inode_dirty(inode);
-		} else {
-			struct allocExtDesc *aed =
-				(struct allocExtDesc *)(epos.bh->b_data);
-			aed->lengthAllocDescs =
-				cpu_to_le32(epos.offset -
-					    sizeof(struct allocExtDesc));
-			if (!UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_STRICT) ||
-			    UDF_SB(inode->i_sb)->s_udfrev >= 0x0201)
-				udf_update_tag(epos.bh->b_data, epos.offset);
-			else
-				udf_update_tag(epos.bh->b_data,
-					       sizeof(struct allocExtDesc));
-			mark_buffer_dirty_inode(epos.bh, inode);
-		}
+		udf_delete_aext(inode, prev_epos);
+		udf_free_blocks(inode->i_sb, inode, &eloc, 0,
+				DIV_ROUND_UP(elen, 1 << inode->i_blkbits));
 	}
 	/* This inode entry is in-memory only and thus we don't have to mark
 	 * the inode dirty */
 	iinfo->i_lenExtents = lbcount;
 	brelse(epos.bh);
+	brelse(prev_epos.bh);
 }
 
 static void udf_update_alloc_ext_desc(struct inode *inode,
diff --git a/kernel/include/asm-generic/atomic.h b/kernel/include/asm-generic/atomic.h
index 11f96f40f4..34100ba982 100644
--- a/kernel/include/asm-generic/atomic.h
+++ b/kernel/include/asm-generic/atomic.h
@@ -76,9 +76,9 @@ static inline void atomic_##op(int i, atomic_t *v)			\
 {									\
 	unsigned long flags;						\
 									\
-	raw_local_irq_save(flags);					\
+	flags = hard_local_irq_save();					\
 	v->counter = v->counter c_op i;					\
-	raw_local_irq_restore(flags);					\
+	hard_local_irq_restore(flags);					\
 }
 
 #define ATOMIC_OP_RETURN(op, c_op)					\
@@ -87,9 +87,9 @@ static inline int atomic_##op##_return(int i, atomic_t *v)		\
 	unsigned long flags;						\
 	int ret;							\
 									\
-	raw_local_irq_save(flags);					\
+	flags = hard_local_irq_save();					\
 	ret = (v->counter = v->counter c_op i);				\
-	raw_local_irq_restore(flags);					\
+	hard_local_irq_restore(flags);					\
 									\
 	return ret;							\
 }
@@ -100,10 +100,10 @@ static inline int atomic_fetch_##op(int i, atomic_t *v)			\
 	unsigned long flags;						\
 	int ret;							\
 									\
-	raw_local_irq_save(flags);					\
+	flags = hard_local_irq_save();					\
 	ret = v->counter;						\
 	v->counter = v->counter c_op i;					\
-	raw_local_irq_restore(flags);					\
+	hard_local_irq_restore(flags);					\
 									\
 	return ret;							\
 }
diff --git a/kernel/include/asm-generic/cmpxchg-local.h b/kernel/include/asm-generic/cmpxchg-local.h
index f17f14f84d..67d712ff0f 100644
--- a/kernel/include/asm-generic/cmpxchg-local.h
+++ b/kernel/include/asm-generic/cmpxchg-local.h
@@ -23,7 +23,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
 	if (size == 8 && sizeof(unsigned long) != 8)
 		wrong_size_cmpxchg(ptr);
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	switch (size) {
 	case 1: prev = *(u8 *)ptr;
 		if (prev == old)
@@ -44,7 +44,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
 	default:
 		wrong_size_cmpxchg(ptr);
 	}
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	return prev;
 }
 
@@ -57,11 +57,11 @@ static inline u64 __cmpxchg64_local_generic(volatile void *ptr,
 	u64 prev;
 	unsigned long flags;
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	prev = *(u64 *)ptr;
 	if (prev == old)
 		*(u64 *)ptr = new;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	return prev;
 }
 
diff --git a/kernel/include/asm-generic/cmpxchg.h b/kernel/include/asm-generic/cmpxchg.h
index 9a24510cd8..475206bd5c 100644
--- a/kernel/include/asm-generic/cmpxchg.h
+++ b/kernel/include/asm-generic/cmpxchg.h
@@ -32,10 +32,10 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
 #ifdef __xchg_u8
 		return __xchg_u8(x, ptr);
 #else
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 		ret = *(volatile u8 *)ptr;
 		*(volatile u8 *)ptr = x;
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		return ret;
 #endif /* __xchg_u8 */
 
@@ -43,10 +43,10 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
 #ifdef __xchg_u16
 		return __xchg_u16(x, ptr);
 #else
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 		ret = *(volatile u16 *)ptr;
 		*(volatile u16 *)ptr = x;
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		return ret;
 #endif /* __xchg_u16 */
 
@@ -54,10 +54,10 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
 #ifdef __xchg_u32
 		return __xchg_u32(x, ptr);
 #else
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 		ret = *(volatile u32 *)ptr;
 		*(volatile u32 *)ptr = x;
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		return ret;
 #endif /* __xchg_u32 */
 
@@ -66,10 +66,10 @@ unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
 #ifdef __xchg_u64
 		return __xchg_u64(x, ptr);
 #else
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 		ret = *(volatile u64 *)ptr;
 		*(volatile u64 *)ptr = x;
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		return ret;
 #endif /* __xchg_u64 */
 #endif /* CONFIG_64BIT */
diff --git a/kernel/include/asm-generic/irq_pipeline.h b/kernel/include/asm-generic/irq_pipeline.h
new file mode 100644
index 0000000000..0f81ed0e73
--- /dev/null
+++ b/kernel/include/asm-generic/irq_pipeline.h
@@ -0,0 +1,109 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#ifndef __ASM_GENERIC_IRQ_PIPELINE_H
+#define __ASM_GENERIC_IRQ_PIPELINE_H
+
+#include <linux/kconfig.h>
+#include <linux/types.h>
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+unsigned long inband_irq_save(void);
+void inband_irq_restore(unsigned long flags);
+void inband_irq_enable(void);
+void inband_irq_disable(void);
+int inband_irqs_disabled(void);
+
+#define hard_cond_local_irq_enable()		hard_local_irq_enable()
+#define hard_cond_local_irq_disable()		hard_local_irq_disable()
+#define hard_cond_local_irq_save()		hard_local_irq_save()
+#define hard_cond_local_irq_restore(__flags)	hard_local_irq_restore(__flags)
+
+#define hard_local_irq_save()			native_irq_save()
+#define hard_local_irq_restore(__flags)		native_irq_restore(__flags)
+#define hard_local_irq_enable()			native_irq_enable()
+#define hard_local_irq_disable()		native_irq_disable()
+#define hard_local_save_flags()			native_save_flags()
+
+#define hard_irqs_disabled()			native_irqs_disabled()
+#define hard_irqs_disabled_flags(__flags)	native_irqs_disabled_flags(__flags)
+
+void irq_pipeline_nmi_enter(void);
+void irq_pipeline_nmi_exit(void);
+
+/* Swap then merge virtual and hardware interrupt states. */
+#define irqs_merge_flags(__flags, __stalled)				\
+	({								\
+		unsigned long __combo =					\
+			arch_irqs_virtual_to_native_flags(__stalled) |	\
+			arch_irqs_native_to_virtual_flags(__flags);	\
+		__combo;						\
+	})
+
+/* Extract swap virtual and hardware interrupt states. */
+#define irqs_split_flags(__combo, __stall_r)				\
+	({								\
+		unsigned long __virt = (__combo);			\
+		*(__stall_r) = hard_irqs_disabled_flags(__combo);	\
+		__virt &= ~arch_irqs_virtual_to_native_flags(*(__stall_r)); \
+		arch_irqs_virtual_to_native_flags(__virt);		\
+	})
+
+#define hard_local_irq_sync()			native_irq_sync()
+
+#else /* !CONFIG_IRQ_PIPELINE */
+
+#define hard_local_save_flags()			({ unsigned long __flags; \
+						raw_local_save_flags(__flags); __flags; })
+#define hard_local_irq_enable()			raw_local_irq_enable()
+#define hard_local_irq_disable()		raw_local_irq_disable()
+#define hard_local_irq_save()			({ unsigned long __flags; \
+						raw_local_irq_save(__flags); __flags; })
+#define hard_local_irq_restore(__flags)		raw_local_irq_restore(__flags)
+
+#define hard_cond_local_irq_enable()		do { } while(0)
+#define hard_cond_local_irq_disable()		do { } while(0)
+#define hard_cond_local_irq_save()		0
+#define hard_cond_local_irq_restore(__flags)	do { (void)(__flags); } while(0)
+
+#define hard_irqs_disabled()			irqs_disabled()
+#define hard_irqs_disabled_flags(__flags)	raw_irqs_disabled_flags(__flags)
+
+static inline void irq_pipeline_nmi_enter(void) { }
+static inline void irq_pipeline_nmi_exit(void) { }
+
+#define hard_local_irq_sync()			do { } while (0)
+
+#endif /* !CONFIG_IRQ_PIPELINE */
+
+#ifdef CONFIG_DEBUG_IRQ_PIPELINE
+void check_inband_stage(void);
+#define check_hard_irqs_disabled()		\
+	WARN_ON_ONCE(!hard_irqs_disabled())
+#else
+static inline void check_inband_stage(void) { }
+static inline int check_hard_irqs_disabled(void) { return 0; }
+#endif
+
+extern bool irq_pipeline_oopsing;
+
+static __always_inline bool irqs_pipelined(void)
+{
+	return IS_ENABLED(CONFIG_IRQ_PIPELINE);
+}
+
+static __always_inline bool irq_pipeline_debug(void)
+{
+	return IS_ENABLED(CONFIG_DEBUG_IRQ_PIPELINE) &&
+		!irq_pipeline_oopsing;
+}
+
+static __always_inline bool irq_pipeline_debug_locking(void)
+{
+	return IS_ENABLED(CONFIG_DEBUG_HARD_LOCKS);
+}
+
+#endif /* __ASM_GENERIC_IRQ_PIPELINE_H */
diff --git a/kernel/include/asm-generic/percpu.h b/kernel/include/asm-generic/percpu.h
index 6432a7fade..8a35f48168 100644
--- a/kernel/include/asm-generic/percpu.h
+++ b/kernel/include/asm-generic/percpu.h
@@ -125,9 +125,9 @@ do {									\
 ({									\
 	typeof(pcp) ___ret;						\
 	unsigned long ___flags;						\
-	raw_local_irq_save(___flags);					\
+	___flags = hard_local_irq_save();				\
 	___ret = raw_cpu_generic_read(pcp);				\
-	raw_local_irq_restore(___flags);				\
+	hard_local_irq_restore(___flags);				\
 	___ret;								\
 })
 
@@ -144,9 +144,9 @@ do {									\
 #define this_cpu_generic_to_op(pcp, val, op)				\
 do {									\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	raw_cpu_generic_to_op(pcp, val, op);				\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 } while (0)
 
 
@@ -154,9 +154,9 @@ do {									\
 ({									\
 	typeof(pcp) __ret;						\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	__ret = raw_cpu_generic_add_return(pcp, val);			\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 	__ret;								\
 })
 
@@ -164,9 +164,9 @@ do {									\
 ({									\
 	typeof(pcp) __ret;						\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	__ret = raw_cpu_generic_xchg(pcp, nval);			\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 	__ret;								\
 })
 
@@ -174,9 +174,9 @@ do {									\
 ({									\
 	typeof(pcp) __ret;						\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	__ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);		\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 	__ret;								\
 })
 
@@ -184,10 +184,10 @@ do {									\
 ({									\
 	int __ret;							\
 	unsigned long __flags;						\
-	raw_local_irq_save(__flags);					\
+	__flags = hard_local_irq_save();				\
 	__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,		\
 			oval1, oval2, nval1, nval2);			\
-	raw_local_irq_restore(__flags);					\
+	hard_local_irq_restore(__flags);				\
 	__ret;								\
 })
 
diff --git a/kernel/include/dovetail/irq.h b/kernel/include/dovetail/irq.h
new file mode 100644
index 0000000000..ac8b5310e2
--- /dev/null
+++ b/kernel/include/dovetail/irq.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _DOVETAIL_IRQ_H
+#define _DOVETAIL_IRQ_H
+
+/* Placeholders for pre- and post-IRQ handling. */
+
+static inline void irq_enter_pipeline(void) { }
+
+static inline void irq_exit_pipeline(void) { }
+
+#endif /* !_DOVETAIL_IRQ_H */
diff --git a/kernel/include/dovetail/mm_info.h b/kernel/include/dovetail/mm_info.h
new file mode 100644
index 0000000000..504bd1d875
--- /dev/null
+++ b/kernel/include/dovetail/mm_info.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _DOVETAIL_MM_INFO_H
+#define _DOVETAIL_MM_INFO_H
+
+/*
+ * Placeholder for per-mm state information defined by the co-kernel.
+ */
+
+struct oob_mm_state {
+};
+
+#endif /* !_DOVETAIL_MM_INFO_H */
diff --git a/kernel/include/dovetail/netdevice.h b/kernel/include/dovetail/netdevice.h
new file mode 100644
index 0000000000..06e8205e25
--- /dev/null
+++ b/kernel/include/dovetail/netdevice.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _DOVETAIL_NETDEVICE_H
+#define _DOVETAIL_NETDEVICE_H
+
+/*
+ * Placeholder for per-device state information defined by the
+ * out-of-band network stack.
+ */
+
+struct oob_netdev_state {
+};
+
+#endif /* !_DOVETAIL_NETDEVICE_H */
diff --git a/kernel/include/dovetail/poll.h b/kernel/include/dovetail/poll.h
new file mode 100644
index 0000000000..d15b14f887
--- /dev/null
+++ b/kernel/include/dovetail/poll.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _DOVETAIL_POLL_H
+#define _DOVETAIL_POLL_H
+
+/*
+ * Placeholder for the out-of-band poll operation descriptor.
+ */
+
+struct oob_poll_wait {
+};
+
+#endif /* !_DOVETAIL_POLL_H */
diff --git a/kernel/include/dovetail/spinlock.h b/kernel/include/dovetail/spinlock.h
new file mode 100644
index 0000000000..381031afda
--- /dev/null
+++ b/kernel/include/dovetail/spinlock.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _DOVETAIL_SPINLOCK_H
+#define _DOVETAIL_SPINLOCK_H
+
+/* Placeholders for hard/hybrid spinlock modifiers. */
+
+struct raw_spinlock;
+
+static inline void hard_spin_lock_prepare(struct raw_spinlock *lock)
+{ }
+
+static inline void hard_spin_unlock_finish(struct raw_spinlock *lock)
+{ }
+
+static inline void hard_spin_trylock_prepare(struct raw_spinlock *lock)
+{ }
+
+static inline void hard_spin_trylock_fail(struct raw_spinlock *lock)
+{ }
+
+#endif /* !_DOVETAIL_SPINLOCK_H */
diff --git a/kernel/include/dovetail/thread_info.h b/kernel/include/dovetail/thread_info.h
new file mode 100644
index 0000000000..4dea8bf1ec
--- /dev/null
+++ b/kernel/include/dovetail/thread_info.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _DOVETAIL_THREAD_INFO_H
+#define _DOVETAIL_THREAD_INFO_H
+
+/*
+ * Placeholder for per-thread state information defined by the
+ * co-kernel.
+ */
+
+struct oob_thread_state {
+};
+
+#endif /* !_DOVETAIL_THREAD_INFO_H */
diff --git a/kernel/include/linux/clockchips.h b/kernel/include/linux/clockchips.h
index 8ae9a95ebf..bda5d7df5d 100644
--- a/kernel/include/linux/clockchips.h
+++ b/kernel/include/linux/clockchips.h
@@ -15,6 +15,7 @@
 # include <linux/cpumask.h>
 # include <linux/ktime.h>
 # include <linux/notifier.h>
+# include <linux/irqstage.h>
 
 struct clock_event_device;
 struct module;
@@ -31,6 +32,7 @@ struct module;
  *		from DETACHED or SHUTDOWN.
  * ONESHOT_STOPPED: Device was programmed in ONESHOT mode and is temporarily
  *		    stopped.
+ * RESERVED:	Device is controlled by an out-of-band core via a proxy.
  */
 enum clock_event_state {
 	CLOCK_EVT_STATE_DETACHED,
@@ -38,6 +40,7 @@ enum clock_event_state {
 	CLOCK_EVT_STATE_PERIODIC,
 	CLOCK_EVT_STATE_ONESHOT,
 	CLOCK_EVT_STATE_ONESHOT_STOPPED,
+	CLOCK_EVT_STATE_RESERVED,
 };
 
 /*
@@ -67,6 +70,17 @@ enum clock_event_state {
  */
 # define CLOCK_EVT_FEAT_HRTIMER		0x000080
 
+/*
+ * Interrupt pipeline support:
+ *
+ * - Clockevent device can work with pipelined timer events (i.e. proxied).
+ * - Device currently delivers high-precision events via out-of-band interrupts.
+ * - Device acts as a proxy for timer interrupt pipelining.
+ */
+# define CLOCK_EVT_FEAT_PIPELINE	0x000100
+# define CLOCK_EVT_FEAT_OOB		0x000200
+# define CLOCK_EVT_FEAT_PROXY		0x000400
+
 /**
  * struct clock_event_device - clock event device descriptor
  * @event_handler:	Assigned by the framework to be called by the low
@@ -91,7 +105,7 @@ enum clock_event_state {
  * @max_delta_ticks:	maximum delta value in ticks stored for reconfiguration
  * @name:		ptr to clock event name
  * @rating:		variable to rate clock event devices
- * @irq:		IRQ number (only for non CPU local devices)
+ * @irq:		IRQ number (only for non CPU local devices, or pipelined timers)
  * @bound_on:		Bound on CPU
  * @cpumask:		cpumask to indicate for which CPUs this device works
  * @list:		list head for the management code
@@ -137,6 +151,11 @@ static inline bool clockevent_state_detached(struct clock_event_device *dev)
 	return dev->state_use_accessors == CLOCK_EVT_STATE_DETACHED;
 }
 
+static inline bool clockevent_state_reserved(struct clock_event_device *dev)
+{
+	return dev->state_use_accessors == CLOCK_EVT_STATE_RESERVED;
+}
+
 static inline bool clockevent_state_shutdown(struct clock_event_device *dev)
 {
 	return dev->state_use_accessors == CLOCK_EVT_STATE_SHUTDOWN;
@@ -157,6 +176,11 @@ static inline bool clockevent_state_oneshot_stopped(struct clock_event_device *d
 	return dev->state_use_accessors == CLOCK_EVT_STATE_ONESHOT_STOPPED;
 }
 
+static inline bool clockevent_is_oob(struct clock_event_device *dev)
+{
+	return !!(dev->features & CLOCK_EVT_FEAT_OOB);
+}
+
 /*
  * Calculate a multiplication factor for scaled math, which is used to convert
  * nanoseconds based values to clock ticks:
@@ -186,6 +210,8 @@ extern int clockevents_unbind_device(struct clock_event_device *ced, int cpu);
 extern void clockevents_config_and_register(struct clock_event_device *dev,
 					    u32 freq, unsigned long min_delta,
 					    unsigned long max_delta);
+extern void clockevents_switch_state(struct clock_event_device *dev,
+				     enum clock_event_state state);
 
 extern int clockevents_update_freq(struct clock_event_device *ce, u32 freq);
 
@@ -215,6 +241,49 @@ static inline int tick_check_broadcast_expired(void) { return 0; }
 static inline void tick_setup_hrtimer_broadcast(void) { }
 # endif
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+struct clock_proxy_device {
+	struct clock_event_device proxy_device;
+	struct clock_event_device *real_device;
+	void (*handle_oob_event)(struct clock_event_device *dev);
+	void (*__setup_handler)(struct clock_proxy_device *dev);
+	void (*__original_handler)(struct clock_event_device *dev);
+};
+
+void tick_notify_proxy(void);
+
+static inline
+void clockevents_handle_event(struct clock_event_device *ced)
+{
+	/*
+	 * If called from the in-band stage, or for delivering a
+	 * high-precision timer event to the out-of-band stage, call
+	 * the event handler immediately.
+	 *
+	 * Otherwise, ced is still the in-band tick device for the
+	 * current CPU, so just relay the incoming tick to the in-band
+	 * stage via tick_notify_proxy().  This situation can happen
+	 * when all CPUs receive the same out-of-band IRQ from a given
+	 * clock event device, but only a subset of the online CPUs has
+	 * enabled a proxy.
+	 */
+	if (clockevent_is_oob(ced) || running_inband())
+		ced->event_handler(ced);
+	else
+		tick_notify_proxy();
+}
+
+#else
+
+static inline
+void clockevents_handle_event(struct clock_event_device *ced)
+{
+	ced->event_handler(ced);
+}
+
+#endif	/* !CONFIG_IRQ_PIPELINE */
+
 #else /* !CONFIG_GENERIC_CLOCKEVENTS: */
 
 static inline void clockevents_suspend(void) { }
diff --git a/kernel/include/linux/clocksource.h b/kernel/include/linux/clocksource.h
index 83a3ebff74..96659740b3 100644
--- a/kernel/include/linux/clocksource.h
+++ b/kernel/include/linux/clocksource.h
@@ -13,12 +13,15 @@
 #include <linux/timex.h>
 #include <linux/time.h>
 #include <linux/list.h>
+#include <linux/hashtable.h>
 #include <linux/cache.h>
 #include <linux/timer.h>
+#include <linux/cdev.h>
 #include <linux/init.h>
 #include <linux/of.h>
 #include <asm/div64.h>
 #include <asm/io.h>
+#include <uapi/linux/clocksource.h>
 
 struct clocksource;
 struct module;
@@ -28,8 +31,15 @@ struct module;
 #include <asm/clocksource.h>
 #endif
 
+
 #include <vdso/clocksource.h>
 
+enum clocksource_vdso_type {
+	CLOCKSOURCE_VDSO_NONE = 0,
+	CLOCKSOURCE_VDSO_ARCHITECTED,
+	CLOCKSOURCE_VDSO_MMIO,	/* <= Must be last. */
+};
+
 /**
  * struct clocksource - hardware abstraction for a free running counter
  *	Provides mostly state-free accessors to the underlying hardware.
@@ -101,6 +111,7 @@ struct clocksource {
 	struct list_head	list;
 	int			rating;
 	enum vdso_clock_mode	vdso_clock_mode;
+  	enum clocksource_vdso_type vdso_type;
 	unsigned long		flags;
 
 	int			(*enable)(struct clocksource *cs);
@@ -120,6 +131,36 @@ struct clocksource {
 	struct module		*owner;
 };
 
+struct clocksource_mmio {
+	void __iomem *reg;
+	struct clocksource clksrc;
+};
+
+struct clocksource_user_mmio {
+	struct clocksource_mmio mmio;
+	void __iomem *reg_upper;
+	unsigned int bits_lower;
+	unsigned int mask_lower;
+	unsigned int mask_upper;
+	enum clksrc_user_mmio_type type;
+	unsigned long phys_lower;
+	unsigned long phys_upper;
+	unsigned int id;
+	struct device *dev;
+	struct cdev cdev;
+	DECLARE_HASHTABLE(mappings, 10);
+	struct spinlock lock;
+	struct list_head link;
+};
+
+struct clocksource_mmio_regs {
+	void __iomem *reg_upper;
+	void __iomem *reg_lower;
+	unsigned int bits_upper;
+	unsigned int bits_lower;
+	unsigned long (*revmap)(void *);
+};
+
 /*
  * Clock source flags bits::
  */
@@ -264,10 +305,21 @@ extern u64 clocksource_mmio_readl_up(struct clocksource *);
 extern u64 clocksource_mmio_readl_down(struct clocksource *);
 extern u64 clocksource_mmio_readw_up(struct clocksource *);
 extern u64 clocksource_mmio_readw_down(struct clocksource *);
+extern u64 clocksource_dual_mmio_readw_up(struct clocksource *);
+extern u64 clocksource_dual_mmio_readl_up(struct clocksource *);
 
 extern int clocksource_mmio_init(void __iomem *, const char *,
 	unsigned long, int, unsigned, u64 (*)(struct clocksource *));
 
+extern int clocksource_user_mmio_init(struct clocksource_user_mmio *ucs,
+				      const struct clocksource_mmio_regs *regs,
+				      unsigned long hz);
+
+extern int clocksource_user_single_mmio_init(
+	void __iomem *base, const char *name,
+	unsigned long hz, int rating, unsigned int bits,
+	u64 (*read)(struct clocksource *));
+
 extern int clocksource_i8253_init(void);
 
 #define TIMER_OF_DECLARE(name, compat, fn) \
diff --git a/kernel/include/linux/console.h b/kernel/include/linux/console.h
index 4b1e26c4cb..1413a45584 100644
--- a/kernel/include/linux/console.h
+++ b/kernel/include/linux/console.h
@@ -141,6 +141,7 @@ static inline int con_debug_leave(void)
 struct console {
 	char	name[16];
 	void	(*write)(struct console *, const char *, unsigned);
+	void	(*write_raw)(struct console *, const char *, unsigned);
 	int	(*read)(struct console *, char *, unsigned);
 	struct tty_driver *(*device)(struct console *, int *);
 	void	(*unblank)(void);
diff --git a/kernel/include/linux/context_tracking_state.h b/kernel/include/linux/context_tracking_state.h
index 65a60d3313..814b57d112 100644
--- a/kernel/include/linux/context_tracking_state.h
+++ b/kernel/include/linux/context_tracking_state.h
@@ -28,7 +28,7 @@ DECLARE_PER_CPU(struct context_tracking, context_tracking);
 
 static __always_inline bool context_tracking_enabled(void)
 {
-	return static_branch_unlikely(&context_tracking_key);
+	return static_branch_unlikely(&context_tracking_key) && running_inband();
 }
 
 static __always_inline bool context_tracking_enabled_cpu(int cpu)
diff --git a/kernel/include/linux/dmaengine.h b/kernel/include/linux/dmaengine.h
index 08537ef11a..e8bc400ab9 100644
--- a/kernel/include/linux/dmaengine.h
+++ b/kernel/include/linux/dmaengine.h
@@ -62,6 +62,7 @@ enum dma_transaction_type {
 	DMA_ASYNC_TX,
 	DMA_SLAVE,
 	DMA_CYCLIC,
+	DMA_OOB,
 	DMA_INTERLEAVE,
 	DMA_COMPLETION_NO_ORDER,
 	DMA_REPEAT,
@@ -191,6 +192,13 @@ struct dma_interleaved_template {
  *  transaction is marked with DMA_PREP_REPEAT will cause the new transaction
  *  to never be processed and stay in the issued queue forever. The flag is
  *  ignored if the previous transaction is not a repeated transaction.
+ * @DMA_OOB_INTERRUPT - if DMA_OOB is supported, handle the completion
+ *  interrupt for this transaction from the out-of-band stage (implies
+ *  DMA_PREP_INTERRUPT). This includes calling the completion callback routine
+ *  from such context if defined for the transaction.
+ * @DMA_OOB_PULSE - if DMA_OOB is supported, (slave) transactions on the
+ *  out-of-band channel should be triggered manually by a call to
+ *  dma_pulse_oob() (implies DMA_OOB_INTERRUPT).
  */
 enum dma_ctrl_flags {
 	DMA_PREP_INTERRUPT = (1 << 0),
@@ -203,6 +211,8 @@ enum dma_ctrl_flags {
 	DMA_PREP_CMD = (1 << 7),
 	DMA_PREP_REPEAT = (1 << 8),
 	DMA_PREP_LOAD_EOT = (1 << 9),
+	DMA_OOB_INTERRUPT = (1 << 10),
+	DMA_OOB_PULSE = (1 << 11),
 };
 
 /**
@@ -940,6 +950,7 @@ struct dma_device {
 					    dma_cookie_t cookie,
 					    struct dma_tx_state *txstate);
 	void (*device_issue_pending)(struct dma_chan *chan);
+	int (*device_pulse_oob)(struct dma_chan *chan);
 	void (*device_release)(struct dma_device *dev);
 	/* debugfs support */
 #ifdef CONFIG_DEBUG_FS
@@ -983,6 +994,14 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
 						  dir, flags, NULL);
 }
 
+static inline bool dmaengine_oob_valid(struct dma_chan *chan,
+				unsigned long flags)
+{
+	return !(dovetailing() &&
+		flags & (DMA_OOB_INTERRUPT|DMA_OOB_PULSE) &&
+		!test_bit(DMA_OOB, chan->device->cap_mask.bits));
+}
+
 static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg(
 	struct dma_chan *chan, struct scatterlist *sgl,	unsigned int sg_len,
 	enum dma_transfer_direction dir, unsigned long flags)
@@ -990,6 +1009,9 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg(
 	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)
 		return NULL;
 
+	if (!dmaengine_oob_valid(chan, flags))
+		return NULL;
+
 	return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
 						  dir, flags, NULL);
 }
@@ -1017,6 +1039,9 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_cyclic(
 	if (!chan || !chan->device || !chan->device->device_prep_dma_cyclic)
 		return NULL;
 
+	if (!dmaengine_oob_valid(chan, flags))
+		return NULL;
+
 	return chan->device->device_prep_dma_cyclic(chan, buf_addr, buf_len,
 						period_len, dir, flags);
 }
@@ -1421,6 +1446,22 @@ static inline void dma_async_issue_pending(struct dma_chan *chan)
 	chan->device->device_issue_pending(chan);
 }
 
+/**
+ * dma_pulse_oob - manual trigger of an out-of-band transaction
+ * @chan: target DMA channel
+ *
+ * Trigger the next out-of-band transaction immediately.
+ */
+static inline int dma_pulse_oob(struct dma_chan *chan)
+{
+	int ret = -ENOTSUPP;
+
+	if (chan->device->device_pulse_oob)
+		ret = chan->device->device_pulse_oob(chan);
+
+	return ret;
+}
+
 /**
  * dma_async_is_tx_complete - poll for transaction completion
  * @chan: DMA channel
diff --git a/kernel/include/linux/dovetail.h b/kernel/include/linux/dovetail.h
new file mode 100644
index 0000000000..9dcbfc5dd9
--- /dev/null
+++ b/kernel/include/linux/dovetail.h
@@ -0,0 +1,325 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#ifndef _LINUX_DOVETAIL_H
+#define _LINUX_DOVETAIL_H
+
+#ifdef CONFIG_DOVETAIL
+
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/thread_info.h>
+#include <linux/irqstage.h>
+#include <uapi/asm-generic/dovetail.h>
+#include <asm/dovetail.h>
+
+struct pt_regs;
+struct task_struct;
+struct file;
+struct files_struct;
+
+enum inband_event_type {
+	INBAND_TASK_SIGNAL,
+	INBAND_TASK_MIGRATION,
+	INBAND_TASK_EXIT,
+	INBAND_TASK_RETUSER,
+	INBAND_TASK_PTSTEP,
+	INBAND_TASK_PTSTOP,
+	INBAND_TASK_PTCONT,
+	INBAND_PROCESS_CLEANUP,
+};
+
+struct dovetail_migration_data {
+	struct task_struct *task;
+	int dest_cpu;
+};
+
+struct dovetail_altsched_context {
+	struct task_struct *task;
+	struct mm_struct *active_mm;
+	bool borrowed_mm;
+};
+
+#define protect_inband_mm(__flags)			\
+	do {						\
+		(__flags) = hard_cond_local_irq_save();	\
+		barrier();				\
+	} while (0)					\
+
+#define unprotect_inband_mm(__flags)			\
+	do {						\
+		barrier();				\
+		hard_cond_local_irq_restore(__flags);	\
+	} while (0)					\
+
+void inband_task_init(struct task_struct *p);
+
+int pipeline_syscall(unsigned int nr, struct pt_regs *regs);
+
+void __oob_trap_notify(unsigned int exception,
+		       struct pt_regs *regs);
+
+static __always_inline void oob_trap_notify(unsigned int exception,
+					struct pt_regs *regs)
+{
+	if (running_oob() && !test_thread_local_flags(_TLF_OOBTRAP))
+		__oob_trap_notify(exception, regs);
+}
+
+void __oob_trap_unwind(unsigned int exception,
+		struct pt_regs *regs);
+
+static __always_inline void oob_trap_unwind(unsigned int exception,
+					struct pt_regs *regs)
+{
+	if (test_thread_local_flags(_TLF_OOBTRAP))
+		__oob_trap_unwind(exception, regs);
+}
+
+void inband_event_notify(enum inband_event_type,
+			 void *data);
+
+void inband_clock_was_set(void);
+
+static inline void inband_signal_notify(struct task_struct *p)
+{
+	if (test_ti_local_flags(task_thread_info(p), _TLF_DOVETAIL))
+		inband_event_notify(INBAND_TASK_SIGNAL, p);
+}
+
+static inline void inband_migration_notify(struct task_struct *p, int cpu)
+{
+	if (test_ti_local_flags(task_thread_info(p), _TLF_DOVETAIL)) {
+		struct dovetail_migration_data d = {
+			.task = p,
+			.dest_cpu = cpu,
+		};
+		inband_event_notify(INBAND_TASK_MIGRATION, &d);
+	}
+}
+
+static inline void inband_exit_notify(void)
+{
+	inband_event_notify(INBAND_TASK_EXIT, NULL);
+}
+
+static inline void inband_cleanup_notify(struct mm_struct *mm)
+{
+	/*
+	 * Notify regardless of _TLF_DOVETAIL: current may have
+	 * resources to clean up although it might not be interested
+	 * in other kernel events.
+	 */
+	inband_event_notify(INBAND_PROCESS_CLEANUP, mm);
+}
+
+static inline void inband_ptstop_notify(void)
+{
+	if (test_thread_local_flags(_TLF_DOVETAIL))
+		inband_event_notify(INBAND_TASK_PTSTOP, current);
+}
+
+static inline void inband_ptcont_notify(void)
+{
+	if (test_thread_local_flags(_TLF_DOVETAIL))
+		inband_event_notify(INBAND_TASK_PTCONT, current);
+}
+
+static inline void inband_ptstep_notify(struct task_struct *tracee)
+{
+	if (test_ti_local_flags(task_thread_info(tracee), _TLF_DOVETAIL))
+		inband_event_notify(INBAND_TASK_PTSTEP, tracee);
+}
+
+static inline
+void prepare_inband_switch(struct task_struct *next)
+{
+	struct task_struct *prev = current;
+
+	if (test_ti_local_flags(task_thread_info(next), _TLF_DOVETAIL))
+		__this_cpu_write(irq_pipeline.rqlock_owner, prev);
+}
+
+void inband_retuser_notify(void);
+
+bool inband_switch_tail(void);
+
+void oob_trampoline(void);
+
+void arch_inband_task_init(struct task_struct *p);
+
+int dovetail_start(void);
+
+void dovetail_stop(void);
+
+void dovetail_init_altsched(struct dovetail_altsched_context *p);
+
+void dovetail_start_altsched(void);
+
+void dovetail_stop_altsched(void);
+
+__must_check int dovetail_leave_inband(void);
+
+static inline void dovetail_leave_oob(void)
+{
+	clear_thread_local_flags(_TLF_OOB|_TLF_OFFSTAGE);
+	clear_thread_flag(TIF_MAYDAY);
+}
+
+void dovetail_resume_inband(void);
+
+bool dovetail_context_switch(struct dovetail_altsched_context *out,
+			struct dovetail_altsched_context *in,
+			bool leave_inband);
+
+static inline
+struct oob_thread_state *dovetail_current_state(void)
+{
+	return &current_thread_info()->oob_state;
+}
+
+static inline
+struct oob_thread_state *dovetail_task_state(struct task_struct *p)
+{
+	return &task_thread_info(p)->oob_state;
+}
+
+static inline
+struct oob_mm_state *dovetail_mm_state(void)
+{
+	if (current->flags & PF_KTHREAD)
+		return NULL;
+
+	return &current->mm->oob_state;
+}
+
+void dovetail_call_mayday(struct pt_regs *regs);
+
+static inline void dovetail_send_mayday(struct task_struct *castaway)
+{
+	struct thread_info *ti = task_thread_info(castaway);
+
+	if (test_ti_local_flags(ti, _TLF_DOVETAIL))
+		set_ti_thread_flag(ti, TIF_MAYDAY);
+}
+
+static inline void dovetail_request_ucall(struct task_struct *task)
+{
+	struct thread_info *ti = task_thread_info(task);
+
+	if (test_ti_local_flags(ti, _TLF_DOVETAIL))
+		set_ti_thread_flag(ti, TIF_RETUSER);
+}
+
+static inline void dovetail_clear_ucall(void)
+{
+	if (test_thread_flag(TIF_RETUSER))
+		clear_thread_flag(TIF_RETUSER);
+}
+
+void install_inband_fd(unsigned int fd, struct file *file,
+		       struct files_struct *files);
+
+void uninstall_inband_fd(unsigned int fd, struct file *file,
+			 struct files_struct *files);
+
+void replace_inband_fd(unsigned int fd, struct file *file,
+		       struct files_struct *files);
+
+#else	/* !CONFIG_DOVETAIL */
+
+/* We may have arch-specific placeholders. */
+#include <asm/dovetail.h>
+
+struct files_struct;
+
+#define protect_inband_mm(__flags)	\
+	do { (void)(__flags); } while (0)
+
+#define unprotect_inband_mm(__flags)	\
+	do { (void)(__flags); } while (0)
+
+static inline
+void inband_task_init(struct task_struct *p) { }
+
+static inline void arch_dovetail_exec_prepare(void)
+{ }
+
+/*
+ * Keep the trap helpers as macros, we might not be able to resolve
+ * trap numbers if CONFIG_DOVETAIL is off.
+ */
+#define oob_trap_notify(__exception, __regs)	do { } while (0)
+#define oob_trap_unwind(__exception, __regs)	do { } while (0)
+
+static inline
+int pipeline_syscall(unsigned int nr, struct pt_regs *regs)
+{
+	return 0;
+}
+
+static inline void inband_signal_notify(struct task_struct *p) { }
+
+static inline
+void inband_migration_notify(struct task_struct *p, int cpu) { }
+
+static inline void inband_exit_notify(void) { }
+
+static inline void inband_cleanup_notify(struct mm_struct *mm) { }
+
+static inline void inband_retuser_notify(void) { }
+
+static inline void inband_ptstop_notify(void) { }
+
+static inline void inband_ptcont_notify(void) { }
+
+static inline void inband_ptstep_notify(struct task_struct *tracee) { }
+
+static inline void oob_trampoline(void) { }
+
+static inline void prepare_inband_switch(struct task_struct *next) { }
+
+static inline bool inband_switch_tail(void)
+{
+	/* Matches converse disabling in prepare_task_switch(). */
+	hard_cond_local_irq_enable();
+	return false;
+}
+
+static inline void dovetail_request_ucall(struct task_struct *task) { }
+
+static inline void dovetail_clear_ucall(void) { }
+
+static inline void inband_clock_was_set(void) { }
+
+static inline
+void install_inband_fd(unsigned int fd, struct file *file,
+		       struct files_struct *files) { }
+
+static inline
+void uninstall_inband_fd(unsigned int fd, struct file *file,
+			 struct files_struct *files) { }
+
+static inline
+void replace_inband_fd(unsigned int fd, struct file *file,
+		       struct files_struct *files) { }
+
+#endif	/* !CONFIG_DOVETAIL */
+
+static __always_inline bool dovetailing(void)
+{
+	return IS_ENABLED(CONFIG_DOVETAIL);
+}
+
+static __always_inline bool dovetail_debug(void)
+{
+	return IS_ENABLED(CONFIG_DEBUG_DOVETAIL);
+}
+
+#ifndef arch_dovetail_is_syscall
+#define arch_dovetail_is_syscall(__nr)	((__nr) == __NR_prctl)
+#endif
+
+#endif /* _LINUX_DOVETAIL_H */
diff --git a/kernel/include/linux/dw_apb_timer.h b/kernel/include/linux/dw_apb_timer.h
index 82ebf92239..d69dbd0961 100644
--- a/kernel/include/linux/dw_apb_timer.h
+++ b/kernel/include/linux/dw_apb_timer.h
@@ -30,7 +30,7 @@ struct dw_apb_clock_event_device {
 
 struct dw_apb_clocksource {
 	struct dw_apb_timer			timer;
-	struct clocksource			cs;
+	struct clocksource_user_mmio		ummio;
 };
 
 void dw_apb_clockevent_register(struct dw_apb_clock_event_device *dw_ced);
diff --git a/kernel/include/linux/entry-common.h b/kernel/include/linux/entry-common.h
index 46c42479f9..3d7d78cf86 100644
--- a/kernel/include/linux/entry-common.h
+++ b/kernel/include/linux/entry-common.h
@@ -72,6 +72,14 @@
 	 _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL |	\
 	 ARCH_EXIT_TO_USER_MODE_WORK)
 
+/*
+ * Status codes of syscall entry when Dovetail is enabled. Must not
+ * conflict with valid syscall numbers. And with -1 which seccomp uses
+ * to skip an syscall.
+ */
+#define EXIT_SYSCALL_OOB	(-2)
+#define EXIT_SYSCALL_TAIL	(-3)
+
 /**
  * arch_check_user_regs - Architecture specific sanity check for user mode regs
  * @regs:	Pointer to currents pt_regs
@@ -181,7 +189,7 @@ static inline void local_irq_enable_exit_to_user(unsigned long ti_work);
 #ifndef local_irq_enable_exit_to_user
 static inline void local_irq_enable_exit_to_user(unsigned long ti_work)
 {
-	local_irq_enable();
+	local_irq_enable_full();
 }
 #endif
 
@@ -196,7 +204,7 @@ static inline void local_irq_disable_exit_to_user(void);
 #ifndef local_irq_disable_exit_to_user
 static inline void local_irq_disable_exit_to_user(void)
 {
-	local_irq_disable();
+	local_irq_disable_full();
 }
 #endif
 
@@ -341,6 +349,12 @@ void irqentry_enter_from_user_mode(struct pt_regs *regs);
  */
 void irqentry_exit_to_user_mode(struct pt_regs *regs);
 
+enum irqentry_info {
+	IRQENTRY_INBAND_UNSTALLED = 0,
+	IRQENTRY_INBAND_STALLED,
+	IRQENTRY_OOB,
+};
+
 #ifndef irqentry_state
 /**
  * struct irqentry_state - Opaque object for exception state storage
@@ -348,6 +362,7 @@ void irqentry_exit_to_user_mode(struct pt_regs *regs);
  *            exit path has to invoke rcu_irq_exit().
  * @lockdep: Used exclusively in the irqentry_nmi_*() calls; ensures that
  *           lockdep state is restored correctly on exit from nmi.
+ * @stage_info: Information about pipeline state and current stage on IRQ entry.
  *
  * This opaque object is filled in by the irqentry_*_enter() functions and
  * must be passed back into the corresponding irqentry_*_exit() functions
@@ -362,6 +377,9 @@ typedef struct irqentry_state {
 		bool	exit_rcu;
 		bool	lockdep;
 	};
+#ifdef CONFIG_IRQ_PIPELINE
+	enum irqentry_info stage_info;
+#endif
 } irqentry_state_t;
 #endif
 
diff --git a/kernel/include/linux/fcntl.h b/kernel/include/linux/fcntl.h
index 766fcd973b..5cb2aa293d 100644
--- a/kernel/include/linux/fcntl.h
+++ b/kernel/include/linux/fcntl.h
@@ -10,7 +10,7 @@
 	(O_RDONLY | O_WRONLY | O_RDWR | O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC | \
 	 O_APPEND | O_NDELAY | O_NONBLOCK | __O_SYNC | O_DSYNC | \
 	 FASYNC	| O_DIRECT | O_LARGEFILE | O_DIRECTORY | O_NOFOLLOW | \
-	 O_NOATIME | O_CLOEXEC | O_PATH | __O_TMPFILE)
+	 O_NOATIME | O_CLOEXEC | O_PATH | __O_TMPFILE | O_OOB)
 
 /* List of all valid flags for the how->upgrade_mask argument: */
 #define VALID_UPGRADE_FLAGS \
diff --git a/kernel/include/linux/fs.h b/kernel/include/linux/fs.h
index 72977650dd..d167e4301b 100644
--- a/kernel/include/linux/fs.h
+++ b/kernel/include/linux/fs.h
@@ -56,6 +56,7 @@ struct kiocb;
 struct kobject;
 struct pipe_inode_info;
 struct poll_table_struct;
+struct oob_poll_wait;
 struct kstatfs;
 struct vm_area_struct;
 struct vfsmount;
@@ -963,6 +964,7 @@ struct file {
 #endif
 	/* needed for tty driver, and maybe others */
 	void			*private_data;
+	void			*oob_data;
 
 #ifdef CONFIG_EPOLL
 	/* Used by fs/eventpoll.c to link all the hooks to this file */
@@ -1800,8 +1802,11 @@ extern long vfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 #ifdef CONFIG_COMPAT
 extern long compat_ptr_ioctl(struct file *file, unsigned int cmd,
 					unsigned long arg);
+extern long compat_ptr_oob_ioctl(struct file *file, unsigned int cmd,
+				 unsigned long arg);
 #else
 #define compat_ptr_ioctl NULL
+#define compat_ptr_oob_ioctl NULL
 #endif
 
 /*
@@ -1888,6 +1893,11 @@ struct file_operations {
 	__poll_t (*poll) (struct file *, struct poll_table_struct *);
 	long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
 	long (*compat_ioctl) (struct file *, unsigned int, unsigned long);
+	ssize_t (*oob_read) (struct file *, char __user *, size_t);
+	ssize_t (*oob_write) (struct file *, const char __user *, size_t);
+	long (*oob_ioctl) (struct file *, unsigned int, unsigned long);
+	long (*compat_oob_ioctl) (struct file *, unsigned int, unsigned long);
+	__poll_t (*oob_poll) (struct file *, struct oob_poll_wait *);
 	int (*mmap) (struct file *, struct vm_area_struct *);
 	unsigned long mmap_supported_flags;
 	int (*open) (struct inode *, struct file *);
diff --git a/kernel/include/linux/hardirq.h b/kernel/include/linux/hardirq.h
index 754f67ac43..955b6ce465 100644
--- a/kernel/include/linux/hardirq.h
+++ b/kernel/include/linux/hardirq.h
@@ -7,6 +7,7 @@
 #include <linux/lockdep.h>
 #include <linux/ftrace_irq.h>
 #include <linux/vtime.h>
+#include <asm-generic/irq_pipeline.h>
 #include <asm/hardirq.h>
 
 extern void synchronize_irq(unsigned int irq);
@@ -122,6 +123,7 @@ extern void rcu_nmi_exit(void);
 
 #define nmi_enter()						\
 	do {							\
+		irq_pipeline_nmi_enter();			\
 		__nmi_enter();					\
 		lockdep_hardirq_enter();			\
 		rcu_nmi_enter();				\
@@ -147,6 +149,22 @@ extern void rcu_nmi_exit(void);
 		rcu_nmi_exit();					\
 		lockdep_hardirq_exit();				\
 		__nmi_exit();					\
+		irq_pipeline_nmi_exit();			\
 	} while (0)
 
+static inline bool start_irq_flow(void)
+{
+	return !irqs_pipelined() || in_pipeline();
+}
+
+static inline bool on_pipeline_entry(void)
+{
+	return irqs_pipelined() && in_pipeline();
+}
+
+static inline bool in_hard_irq(void)
+{
+	return irqs_pipelined() ? in_pipeline() : in_irq();
+}
+
 #endif /* LINUX_HARDIRQ_H */
diff --git a/kernel/include/linux/intel-iommu.h b/kernel/include/linux/intel-iommu.h
index 142ec79cda..c1be3c0559 100644
--- a/kernel/include/linux/intel-iommu.h
+++ b/kernel/include/linux/intel-iommu.h
@@ -576,7 +576,7 @@ struct intel_iommu {
 	u64		ecap;
 	u64		vccap;
 	u32		gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */
-	raw_spinlock_t	register_lock; /* protect register handling */
+	hard_spinlock_t	register_lock; /* protect register handling */
 	int		seq_id;	/* sequence id of the iommu */
 	int		agaw; /* agaw of this iommu */
 	int		msagaw; /* max sagaw of this iommu */
diff --git a/kernel/include/linux/interrupt.h b/kernel/include/linux/interrupt.h
index 386ddf43e2..c89728ff43 100644
--- a/kernel/include/linux/interrupt.h
+++ b/kernel/include/linux/interrupt.h
@@ -61,6 +61,12 @@
  *                interrupt handler after suspending interrupts. For system
  *                wakeup devices users need to implement wakeup detection in
  *                their interrupt handlers.
+ * IRQF_OOB - Interrupt is attached to an out-of-band handler living
+ *            on the heading stage of the interrupt pipeline
+ *            (CONFIG_IRQ_PIPELINE).  It may be delivered to the
+ *            handler any time interrupts are enabled in the CPU,
+ *            regardless of the (virtualized) interrupt state
+ *            maintained by local_irq_save/disable().
  */
 #define IRQF_SHARED		0x00000080
 #define IRQF_PROBE_SHARED	0x00000100
@@ -74,6 +80,7 @@
 #define IRQF_NO_THREAD		0x00010000
 #define IRQF_EARLY_RESUME	0x00020000
 #define IRQF_COND_SUSPEND	0x00040000
+#define IRQF_OOB		0x00080000
 
 #define IRQF_TIMER		(__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD)
 
@@ -514,9 +521,29 @@ extern bool force_irqthreads;
  * to ensure that after a local_irq_disable(), interrupts have
  * really been disabled in hardware. Such architectures need to
  * implement the following hook.
+ *
+ * Those cases also apply when interrupt pipelining is in effect,
+ * since we are virtualizing the interrupt disable state here too.
  */
 #ifndef hard_irq_disable
-#define hard_irq_disable()	do { } while(0)
+#define hard_irq_disable()	hard_cond_local_irq_disable()
+#endif
+
+/*
+ * Unlike other virtualized interrupt disabling schemes may assume, we
+ * can't expect local_irq_restore() to turn hard interrupts on when
+ * pipelining.  hard_irq_enable() is introduced to be paired with
+ * hard_irq_disable(), for unconditionally turning them on. The only
+ * sane sequence mixing virtual and real disable state manipulation
+ * is:
+ *
+ * 1. local_irq_save/disable
+ * 2. hard_irq_disable
+ * 3. hard_irq_enable
+ * 4. local_irq_restore/enable
+ */
+#ifndef hard_irq_enable
+#define hard_irq_enable()	hard_cond_local_irq_enable()
 #endif
 
 /* PLEASE, avoid to allocate new softirqs, if you need not _really_ high
diff --git a/kernel/include/linux/irq.h b/kernel/include/linux/irq.h
index b2b956d0b8..44bd457a30 100644
--- a/kernel/include/linux/irq.h
+++ b/kernel/include/linux/irq.h
@@ -16,6 +16,7 @@
 #include <linux/irqhandler.h>
 #include <linux/irqreturn.h>
 #include <linux/irqnr.h>
+#include <linux/irq_work.h>
 #include <linux/topology.h>
 #include <linux/io.h>
 #include <linux/slab.h>
@@ -73,6 +74,11 @@ enum irqchip_irq_state;
  * IRQ_DISABLE_UNLAZY		- Disable lazy irq disable
  * IRQ_HIDDEN			- Don't show up in /proc/interrupts
  * IRQ_RAW			- Skip tick management and irqtime accounting
+ * IRQ_OOB                      - Interrupt can be delivered to the out-of-band handler
+ *                                when pipelining is enabled (CONFIG_IRQ_PIPELINE),
+ *                                regardless of the (virtualized) interrupt state
+ *                                maintained by local_irq_save/disable().
+ * IRQ_CHAINED                  - Interrupt is chained.
  */
 enum {
 	IRQ_TYPE_NONE		= 0x00000000,
@@ -101,13 +107,15 @@ enum {
 	IRQ_DISABLE_UNLAZY	= (1 << 19),
 	IRQ_HIDDEN		= (1 << 20),
 	IRQ_RAW			= (1 << 21),
+	IRQ_OOB			= (1 << 22),
+	IRQ_CHAINED		= (1 << 23),
 };
 
 #define IRQF_MODIFY_MASK	\
 	(IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \
 	 IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \
 	 IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \
-	 IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY | IRQ_HIDDEN)
+	 IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY | IRQ_HIDDEN | IRQ_OOB)
 
 #define IRQ_NO_BALANCING_MASK	(IRQ_PER_CPU | IRQ_NO_BALANCING)
 
@@ -173,6 +181,7 @@ struct irq_common_data {
  *			irq_domain
  * @chip_data:		platform-specific per-chip private data for the chip
  *			methods, to allow shared chip implementations
+ * @move_work:		irq_work for setaffinity deferral when pipelining irqs
  */
 struct irq_data {
 	u32			mask;
@@ -183,6 +192,9 @@ struct irq_data {
 	struct irq_domain	*domain;
 #ifdef	CONFIG_IRQ_DOMAIN_HIERARCHY
 	struct irq_data		*parent_data;
+#endif
+#if defined(CONFIG_IRQ_PIPELINE) && defined(CONFIG_GENERIC_PENDING_IRQ)
+	struct irq_work		move_work;
 #endif
 	void			*chip_data;
 };
@@ -221,6 +233,7 @@ struct irq_data {
  *				  irq_chip::irq_set_affinity() when deactivated.
  * IRQD_IRQ_ENABLED_ON_SUSPEND	- Interrupt is enabled on suspend by irq pm if
  *				  irqchip have flag IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND set.
+ * IRQD_SETAFFINITY_BLOCKED	- Pending affinity setting on hold (IRQ_PIPELINE)
  */
 enum {
 	IRQD_TRIGGER_MASK		= 0xf,
@@ -247,6 +260,7 @@ enum {
 	IRQD_HANDLE_ENFORCE_IRQCTX	= (1 << 28),
 	IRQD_AFFINITY_ON_ACTIVATE	= (1 << 29),
 	IRQD_IRQ_ENABLED_ON_SUSPEND	= (1 << 30),
+	IRQD_SETAFFINITY_BLOCKED	= (1 << 31),
 };
 
 #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors)
@@ -256,6 +270,21 @@ static inline bool irqd_is_setaffinity_pending(struct irq_data *d)
 	return __irqd_to_state(d) & IRQD_SETAFFINITY_PENDING;
 }
 
+static inline void irqd_set_move_blocked(struct irq_data *d)
+{
+	__irqd_to_state(d) |= IRQD_SETAFFINITY_BLOCKED;
+}
+
+static inline void irqd_clr_move_blocked(struct irq_data *d)
+{
+	__irqd_to_state(d) &= ~IRQD_SETAFFINITY_BLOCKED;
+}
+
+static inline bool irqd_is_setaffinity_blocked(struct irq_data *d)
+{
+	return irqs_pipelined() && __irqd_to_state(d) & IRQD_SETAFFINITY_BLOCKED;
+}
+
 static inline bool irqd_is_per_cpu(struct irq_data *d)
 {
 	return __irqd_to_state(d) & IRQD_PER_CPU;
@@ -570,6 +599,7 @@ struct irq_chip {
  * IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND:  Invokes __enable_irq()/__disable_irq() for wake irqs
  *                                    in the suspend path if they are in disabled state
  * IRQCHIP_AFFINITY_PRE_STARTUP:      Default affinity update before startup
+ * IRQCHIP_PIPELINE_SAFE:             Chip can work in pipelined mode
  */
 enum {
 	IRQCHIP_SET_TYPE_MASKED			= (1 <<  0),
@@ -583,6 +613,7 @@ enum {
 	IRQCHIP_SUPPORTS_NMI			= (1 <<  8),
 	IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND	= (1 <<  9),
 	IRQCHIP_AFFINITY_PRE_STARTUP		= (1 << 10),
+	IRQCHIP_PIPELINE_SAFE			= (1 << 11),
 };
 
 #include <linux/irqdesc.h>
@@ -660,6 +691,7 @@ extern void handle_percpu_irq(struct irq_desc *desc);
 extern void handle_percpu_devid_irq(struct irq_desc *desc);
 extern void handle_bad_irq(struct irq_desc *desc);
 extern void handle_nested_irq(unsigned int irq);
+extern void handle_synthetic_irq(struct irq_desc *desc);
 
 extern void handle_fasteoi_nmi(struct irq_desc *desc);
 extern void handle_percpu_devid_fasteoi_nmi(struct irq_desc *desc);
@@ -813,7 +845,13 @@ extern int irq_set_irq_type(unsigned int irq, unsigned int type);
 extern int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry);
 extern int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
 				struct msi_desc *entry);
-extern struct irq_data *irq_get_irq_data(unsigned int irq);
+
+static inline struct irq_data *irq_get_irq_data(unsigned int irq)
+{
+	struct irq_desc *desc = irq_to_desc(irq);
+
+	return desc ? &desc->irq_data : NULL;
+}
 
 static inline struct irq_chip *irq_get_chip(unsigned int irq)
 {
@@ -1056,7 +1094,7 @@ struct irq_chip_type {
  * different flow mechanisms (level/edge) for it.
  */
 struct irq_chip_generic {
-	raw_spinlock_t		lock;
+	hard_spinlock_t		lock;
 	void __iomem		*reg_base;
 	u32			(*reg_readl)(void __iomem *addr);
 	void			(*reg_writel)(u32 val, void __iomem *addr);
@@ -1183,6 +1221,12 @@ static inline struct irq_chip_type *irq_data_get_chip_type(struct irq_data *d)
 
 #define IRQ_MSK(n) (u32)((n) < 32 ? ((1 << (n)) - 1) : UINT_MAX)
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+int irq_switch_oob(unsigned int irq, bool on);
+
+#endif	/* !CONFIG_IRQ_PIPELINE */
+
 #ifdef CONFIG_SMP
 static inline void irq_gc_lock(struct irq_chip_generic *gc)
 {
diff --git a/kernel/include/linux/irq_pipeline.h b/kernel/include/linux/irq_pipeline.h
new file mode 100644
index 0000000000..cbeb0109fa
--- /dev/null
+++ b/kernel/include/linux/irq_pipeline.h
@@ -0,0 +1,145 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2002 Philippe Gerum  <rpm@xenomai.org>.
+ *               2006 Gilles Chanteperdrix.
+ *               2007 Jan Kiszka.
+ */
+#ifndef _LINUX_IRQ_PIPELINE_H
+#define _LINUX_IRQ_PIPELINE_H
+
+struct cpuidle_device;
+struct cpuidle_state;
+struct irq_desc;
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+#include <linux/compiler.h>
+#include <linux/irqdomain.h>
+#include <linux/percpu.h>
+#include <linux/interrupt.h>
+#include <linux/irqstage.h>
+#include <linux/thread_info.h>
+#include <asm/irqflags.h>
+
+void irq_pipeline_init_early(void);
+
+void irq_pipeline_init(void);
+
+void arch_irq_pipeline_init(void);
+
+void generic_pipeline_irq_desc(struct irq_desc *desc,
+			       struct pt_regs *regs);
+
+int irq_inject_pipeline(unsigned int irq);
+
+void synchronize_pipeline(void);
+
+static __always_inline void synchronize_pipeline_on_irq(void)
+{
+	/*
+	 * Optimize if we preempted the high priority oob stage: we
+	 * don't need to synchronize the pipeline unless there is a
+	 * pending interrupt for it.
+	 */
+	if (running_inband() ||
+	    stage_irqs_pending(this_oob_staged()))
+		synchronize_pipeline();
+}
+
+bool handle_oob_irq(struct irq_desc *desc);
+
+void arch_do_IRQ_pipelined(struct irq_desc *desc);
+
+#ifdef CONFIG_SMP
+void irq_send_oob_ipi(unsigned int ipi,
+		const struct cpumask *cpumask);
+#endif	/* CONFIG_SMP */
+
+void irq_pipeline_oops(void);
+
+bool irq_cpuidle_enter(struct cpuidle_device *dev,
+		       struct cpuidle_state *state);
+
+int run_oob_call(int (*fn)(void *arg), void *arg);
+
+extern bool irq_pipeline_active;
+
+static inline bool inband_unsafe(void)
+{
+	return running_oob() ||
+		(hard_irqs_disabled() && irq_pipeline_active);
+}
+
+static inline bool inband_irq_pending(void)
+{
+	check_hard_irqs_disabled();
+
+	return stage_irqs_pending(this_inband_staged());
+}
+
+struct irq_stage_data *
+handle_irq_pipelined_prepare(struct pt_regs *regs);
+
+int handle_irq_pipelined_finish(struct irq_stage_data *prevd,
+				struct pt_regs *regs);
+
+int handle_irq_pipelined(struct pt_regs *regs);
+
+void sync_inband_irqs(void);
+
+extern struct irq_domain *synthetic_irq_domain;
+
+#else /* !CONFIG_IRQ_PIPELINE */
+
+#include <linux/irqstage.h>
+#include <linux/hardirq.h>
+
+static inline
+void irq_pipeline_init_early(void) { }
+
+static inline
+void irq_pipeline_init(void) { }
+
+static inline
+void irq_pipeline_oops(void) { }
+
+static inline int
+generic_pipeline_irq_desc(struct irq_desc *desc,
+			struct pt_regs *regs)
+{
+	return 0;
+}
+
+static inline bool handle_oob_irq(struct irq_desc *desc)
+{
+	return false;
+}
+
+static inline bool irq_cpuidle_enter(struct cpuidle_device *dev,
+				     struct cpuidle_state *state)
+{
+	return true;
+}
+
+static inline bool inband_unsafe(void)
+{
+	return false;
+}
+
+static inline bool inband_irq_pending(void)
+{
+	return false;
+}
+
+static inline void sync_inband_irqs(void) { }
+
+#endif /* !CONFIG_IRQ_PIPELINE */
+
+#if !defined(CONFIG_IRQ_PIPELINE) || !defined(CONFIG_SPARSE_IRQ)
+static inline void uncache_irq_desc(unsigned int irq) { }
+#else
+void uncache_irq_desc(unsigned int irq);
+#endif
+
+#endif /* _LINUX_IRQ_PIPELINE_H */
diff --git a/kernel/include/linux/irqdesc.h b/kernel/include/linux/irqdesc.h
index 5745491303..f134909335 100644
--- a/kernel/include/linux/irqdesc.h
+++ b/kernel/include/linux/irqdesc.h
@@ -68,7 +68,7 @@ struct irq_desc {
 	unsigned int		irqs_unhandled;
 	atomic_t		threads_handled;
 	int			threads_handled_last;
-	raw_spinlock_t		lock;
+	hybrid_spinlock_t	lock;
 	struct cpumask		*percpu_enabled;
 	const struct cpumask	*percpu_affinity;
 #ifdef CONFIG_SMP
@@ -154,6 +154,8 @@ static inline void generic_handle_irq_desc(struct irq_desc *desc)
 
 int generic_handle_irq(unsigned int irq);
 
+void generic_pipeline_irq(unsigned int irq, struct pt_regs *regs);
+
 #ifdef CONFIG_HANDLE_DOMAIN_IRQ
 /*
  * Convert a HW interrupt number to a logical one using a IRQ domain,
@@ -164,11 +166,26 @@ int generic_handle_irq(unsigned int irq);
 int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq,
 			bool lookup, struct pt_regs *regs);
 
+#ifdef CONFIG_IRQ_PIPELINE
+unsigned int irq_find_mapping(struct irq_domain *host,
+			irq_hw_number_t hwirq);
+
+static inline int handle_domain_irq(struct irq_domain *domain,
+				    unsigned int hwirq, struct pt_regs *regs)
+{
+	unsigned int irq = irq_find_mapping(domain, hwirq);
+
+	generic_pipeline_irq(irq, regs);
+
+	return 0;
+}
+#else
 static inline int handle_domain_irq(struct irq_domain *domain,
 				    unsigned int hwirq, struct pt_regs *regs)
 {
 	return __handle_domain_irq(domain, hwirq, true, regs);
 }
+#endif	/* !CONFIG_IRQ_PIPELINE */
 
 #ifdef CONFIG_IRQ_DOMAIN
 int handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq,
@@ -252,6 +269,14 @@ static inline bool irq_is_percpu_devid(unsigned int irq)
 	return desc->status_use_accessors & IRQ_PER_CPU_DEVID;
 }
 
+static inline int irq_is_oob(unsigned int irq)
+{
+	struct irq_desc *desc;
+
+	desc = irq_to_desc(irq);
+	return desc->status_use_accessors & IRQ_OOB;
+}
+
 static inline void
 irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class,
 		      struct lock_class_key *request_class)
diff --git a/kernel/include/linux/irqflags.h b/kernel/include/linux/irqflags.h
index 3ed4e8771b..051c72751d 100644
--- a/kernel/include/linux/irqflags.h
+++ b/kernel/include/linux/irqflags.h
@@ -13,6 +13,7 @@
 #define _LINUX_TRACE_IRQFLAGS_H
 
 #include <linux/typecheck.h>
+#include <asm-generic/irq_pipeline.h>
 #include <asm/irqflags.h>
 #include <asm/percpu.h>
 
@@ -52,7 +53,9 @@ DECLARE_PER_CPU(int, hardirq_context);
 extern void trace_hardirqs_on_prepare(void);
 extern void trace_hardirqs_off_finish(void);
 extern void trace_hardirqs_on(void);
+extern void trace_hardirqs_on_pipelined(void);
 extern void trace_hardirqs_off(void);
+extern void trace_hardirqs_off_pipelined(void);
 
 # define lockdep_hardirq_context()	(raw_cpu_read(hardirq_context))
 # define lockdep_softirq_context(p)	((p)->softirq_context)
@@ -122,7 +125,9 @@ do {						\
 # define trace_hardirqs_on_prepare()		do { } while (0)
 # define trace_hardirqs_off_finish()		do { } while (0)
 # define trace_hardirqs_on()			do { } while (0)
+# define trace_hardirqs_on_pipelined()		do { } while (0)
 # define trace_hardirqs_off()			do { } while (0)
+# define trace_hardirqs_off_pipelined()		do { } while (0)
 # define lockdep_hardirq_context()		0
 # define lockdep_softirq_context(p)		0
 # define lockdep_hardirqs_enabled()		0
@@ -228,6 +233,38 @@ do {						\
 
 #endif /* CONFIG_TRACE_IRQFLAGS */
 
+#ifdef CONFIG_IRQ_PIPELINE
+#define local_irq_enable_full()			\
+	do {					\
+		hard_local_irq_enable();	\
+		local_irq_enable();		\
+	} while (0)
+
+#define local_irq_disable_full()		\
+	do {					\
+		hard_local_irq_disable();	\
+		local_irq_disable();		\
+	} while (0)
+
+#define local_irq_save_full(__flags)		\
+  	do {					\
+		hard_local_irq_disable();	\
+		local_irq_save(__flags);	\
+	} while (0)
+
+#define local_irq_restore_full(__flags)			\
+	do {						\
+		if (!irqs_disabled_flags(__flags))	\
+			hard_local_irq_enable();	\
+		local_irq_restore(__flags);		\
+	} while (0)
+#else
+#define local_irq_enable_full()		local_irq_enable()
+#define local_irq_disable_full()	local_irq_disable()
+#define local_irq_save_full(__flags)	local_irq_save(__flags)
+#define local_irq_restore_full(__flags)	local_irq_restore(__flags)
+#endif
+
 #define local_save_flags(flags)	raw_local_save_flags(flags)
 
 /*
diff --git a/kernel/include/linux/irqstage.h b/kernel/include/linux/irqstage.h
new file mode 100644
index 0000000000..46bfb84b06
--- /dev/null
+++ b/kernel/include/linux/irqstage.h
@@ -0,0 +1,398 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016, 2019 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#ifndef _LINUX_IRQSTAGE_H
+#define _LINUX_IRQSTAGE_H
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+#include <linux/percpu.h>
+#include <linux/bitops.h>
+#include <linux/preempt.h>
+#include <linux/sched.h>
+#include <asm/irq_pipeline.h>
+
+struct kvm_oob_notifier;
+
+struct irq_stage {
+	int index;
+	const char *name;
+};
+
+extern struct irq_stage inband_stage;
+
+extern struct irq_stage oob_stage;
+
+struct irq_event_map;
+
+struct irq_log {
+	unsigned long index_0;
+	struct irq_event_map *map;
+};
+
+/* Per-CPU, per-stage data. */
+struct irq_stage_data {
+	struct irq_log log;
+	struct irq_stage *stage;
+#ifdef CONFIG_DEBUG_IRQ_PIPELINE
+	int cpu;
+#endif
+};
+
+/* Per-CPU pipeline descriptor. */
+struct irq_pipeline_data {
+	struct irq_stage_data stages[2];
+	struct pt_regs tick_regs;
+#ifdef CONFIG_DOVETAIL
+	struct task_struct *task_inflight;
+	struct task_struct *rqlock_owner;
+#ifdef CONFIG_KVM
+	struct kvm_oob_notifier *vcpu_notify;
+#endif
+#endif
+};
+
+DECLARE_PER_CPU(struct irq_pipeline_data, irq_pipeline);
+
+/*
+ * The low-level stall bit accessors. Should be used by the Dovetail
+ * core implementation exclusively, inband_irq_*() and oob_irq_*()
+ * accessors are available to common code.
+ */
+
+#define INBAND_STALL_BIT  0
+#define OOB_STALL_BIT     1
+
+static __always_inline void init_task_stall_bits(struct task_struct *p)
+{
+	__set_bit(INBAND_STALL_BIT, &p->stall_bits);
+	__clear_bit(OOB_STALL_BIT, &p->stall_bits);
+}
+
+static __always_inline void stall_inband_nocheck(void)
+{
+	__set_bit(INBAND_STALL_BIT, &current->stall_bits);
+	barrier();
+}
+
+static __always_inline void stall_inband(void)
+{
+	WARN_ON_ONCE(irq_pipeline_debug() && running_oob());
+	stall_inband_nocheck();
+}
+
+static __always_inline void unstall_inband_nocheck(void)
+{
+	barrier();
+	__clear_bit(INBAND_STALL_BIT, &current->stall_bits);
+}
+
+static __always_inline void unstall_inband(void)
+{
+	WARN_ON_ONCE(irq_pipeline_debug() && running_oob());
+	unstall_inband_nocheck();
+}
+
+static __always_inline int test_and_stall_inband_nocheck(void)
+{
+	return __test_and_set_bit(INBAND_STALL_BIT, &current->stall_bits);
+}
+
+static __always_inline int test_and_stall_inband(void)
+{
+	WARN_ON_ONCE(irq_pipeline_debug() && running_oob());
+	return test_and_stall_inband_nocheck();
+}
+
+static __always_inline int test_inband_stall(void)
+{
+	return test_bit(INBAND_STALL_BIT, &current->stall_bits);
+}
+
+static __always_inline void stall_oob(void)
+{
+	__set_bit(OOB_STALL_BIT, &current->stall_bits);
+	barrier();
+}
+
+static __always_inline void unstall_oob(void)
+{
+	barrier();
+	__clear_bit(OOB_STALL_BIT, &current->stall_bits);
+}
+
+static __always_inline int test_and_stall_oob(void)
+{
+	return __test_and_set_bit(OOB_STALL_BIT, &current->stall_bits);
+}
+
+static __always_inline int test_oob_stall(void)
+{
+	return test_bit(OOB_STALL_BIT, &current->stall_bits);
+}
+
+/**
+ * this_staged - IRQ stage data on the current CPU
+ *
+ * Return the address of @stage's data on the current CPU. IRQs must
+ * be hard disabled to prevent CPU migration.
+ */
+static __always_inline
+struct irq_stage_data *this_staged(struct irq_stage *stage)
+{
+	return &raw_cpu_ptr(irq_pipeline.stages)[stage->index];
+}
+
+/**
+ * percpu_inband_staged - IRQ stage data on specified CPU
+ *
+ * Return the address of @stage's data on @cpu.
+ *
+ * This is the slowest accessor, use it carefully. Prefer
+ * this_staged() for requests referring to the current
+ * CPU. Additionally, if the target stage is known at build time,
+ * consider using this_{inband, oob}_staged() instead.
+ */
+static __always_inline
+struct irq_stage_data *percpu_inband_staged(struct irq_stage *stage, int cpu)
+{
+	return &per_cpu(irq_pipeline.stages, cpu)[stage->index];
+}
+
+/**
+ * this_inband_staged - return the address of the pipeline context
+ * data for the inband stage on the current CPU. CPU migration must be
+ * disabled.
+ *
+ * This accessor is recommended when the stage we refer to is known at
+ * build time to be the inband one.
+ */
+static __always_inline struct irq_stage_data *this_inband_staged(void)
+{
+	return raw_cpu_ptr(&irq_pipeline.stages[0]);
+}
+
+/**
+ * this_oob_staged - return the address of the pipeline context data
+ * for the registered oob stage on the current CPU. CPU migration must
+ * be disabled.
+ *
+ * This accessor is recommended when the stage we refer to is known at
+ * build time to be the registered oob stage. This address is always
+ * different from the context data of the inband stage, even in
+ * absence of registered oob stage.
+ */
+static __always_inline struct irq_stage_data *this_oob_staged(void)
+{
+	return raw_cpu_ptr(&irq_pipeline.stages[1]);
+}
+
+static __always_inline struct irq_stage_data *__current_irq_staged(void)
+{
+	return &raw_cpu_ptr(irq_pipeline.stages)[stage_level()];
+}
+
+/**
+ * current_irq_staged - return the address of the pipeline context
+ * data for the current stage. CPU migration must be disabled.
+ */
+#define current_irq_staged __current_irq_staged()
+
+static __always_inline
+void check_staged_locality(struct irq_stage_data *pd)
+{
+#ifdef CONFIG_DEBUG_IRQ_PIPELINE
+	/*
+	 * Setting our context with another processor's is a really
+	 * bad idea, our caller definitely went loopy.
+	 */
+	WARN_ON_ONCE(raw_smp_processor_id() != pd->cpu);
+#endif
+}
+
+/**
+ * switch_oob(), switch_inband() - switch the current CPU to the
+ * specified stage context. CPU migration must be disabled.
+ *
+ * Calling these routines is the only sane and safe way to change the
+ * interrupt stage for the current CPU. Don't bypass them, ever.
+ * Really.
+ */
+static __always_inline
+void switch_oob(struct irq_stage_data *pd)
+{
+	check_staged_locality(pd);
+	if (!(preempt_count() & STAGE_MASK))
+		preempt_count_add(STAGE_OFFSET);
+}
+
+static __always_inline
+void switch_inband(struct irq_stage_data *pd)
+{
+	check_staged_locality(pd);
+	if (preempt_count() & STAGE_MASK)
+		preempt_count_sub(STAGE_OFFSET);
+}
+
+static __always_inline
+void set_current_irq_staged(struct irq_stage_data *pd)
+{
+	if (pd->stage == &inband_stage)
+		switch_inband(pd);
+	else
+		switch_oob(pd);
+}
+
+static __always_inline struct irq_stage *__current_irq_stage(void)
+{
+	/*
+	 * We don't have to hard disable irqs while accessing the
+	 * per-CPU stage data here, because there is no way we could
+	 * switch stage and CPU at the same time.
+	 */
+	return __current_irq_staged()->stage;
+}
+
+#define current_irq_stage	__current_irq_stage()
+
+static __always_inline bool oob_stage_present(void)
+{
+	return oob_stage.index != 0;
+}
+
+/**
+ * stage_irqs_pending() - Whether we have interrupts pending
+ * (i.e. logged) on the current CPU for the given stage. Hard IRQs
+ * must be disabled.
+ */
+static __always_inline int stage_irqs_pending(struct irq_stage_data *pd)
+{
+	return pd->log.index_0 != 0;
+}
+
+void sync_current_irq_stage(void);
+
+void sync_irq_stage(struct irq_stage *top);
+
+void irq_post_stage(struct irq_stage *stage,
+		    unsigned int irq);
+
+static __always_inline void irq_post_oob(unsigned int irq)
+{
+	irq_post_stage(&oob_stage, irq);
+}
+
+static __always_inline void irq_post_inband(unsigned int irq)
+{
+	irq_post_stage(&inband_stage, irq);
+}
+
+static __always_inline void oob_irq_disable(void)
+{
+	hard_local_irq_disable();
+	stall_oob();
+}
+
+static __always_inline unsigned long oob_irq_save(void)
+{
+	hard_local_irq_disable();
+	return test_and_stall_oob();
+}
+
+static __always_inline int oob_irqs_disabled(void)
+{
+	return test_oob_stall();
+}
+
+void oob_irq_enable(void);
+
+void __oob_irq_restore(unsigned long x);
+
+static __always_inline void oob_irq_restore(unsigned long x)
+{
+	if ((x ^ test_oob_stall()) & 1)
+		__oob_irq_restore(x);
+}
+
+bool stage_disabled(void);
+
+unsigned long test_and_lock_stage(int *irqsoff);
+
+void unlock_stage(unsigned long irqstate);
+
+#define stage_save_flags(__irqstate)					\
+  	do {								\
+	  unsigned long __flags = hard_local_save_flags();		\
+	  (__irqstate) = irqs_merge_flags(__flags,			\
+					  irqs_disabled());		\
+	} while (0)
+
+int enable_oob_stage(const char *name);
+
+int arch_enable_oob_stage(void);
+
+void disable_oob_stage(void);
+
+#else /* !CONFIG_IRQ_PIPELINE */
+
+#include <linux/irqflags.h>
+
+void call_is_nop_without_pipelining(void);
+
+static __always_inline void stall_inband(void) { }
+
+static __always_inline void unstall_inband(void) { }
+
+static __always_inline int test_and_stall_inband(void)
+{
+	return false;
+}
+
+static __always_inline int test_inband_stall(void)
+{
+	return false;
+}
+
+static __always_inline bool oob_stage_present(void)
+{
+	return false;
+}
+
+static __always_inline bool stage_disabled(void)
+{
+	return irqs_disabled();
+}
+
+static __always_inline void irq_post_inband(unsigned int irq)
+{
+	call_is_nop_without_pipelining();
+}
+
+#define test_and_lock_stage(__irqsoff)				\
+	({							\
+		unsigned long __flags;				\
+		raw_local_irq_save(__flags);			\
+		*(__irqsoff) = irqs_disabled_flags(__flags);	\
+		__flags;					\
+	})
+
+#define unlock_stage(__flags)		raw_local_irq_restore(__flags)
+
+#define stage_save_flags(__flags)	raw_local_save_flags(__flags)
+
+static __always_inline void stall_inband_nocheck(void)
+{ }
+
+static __always_inline void unstall_inband_nocheck(void)
+{ }
+
+static __always_inline int test_and_stall_inband_nocheck(void)
+{
+	return irqs_disabled();
+}
+
+#endif /* !CONFIG_IRQ_PIPELINE */
+
+#endif	/* !_LINUX_IRQSTAGE_H */
diff --git a/kernel/include/linux/kernel.h b/kernel/include/linux/kernel.h
index c333dc6fb1..b45aaf3d83 100644
--- a/kernel/include/linux/kernel.h
+++ b/kernel/include/linux/kernel.h
@@ -15,6 +15,7 @@
 #include <linux/typecheck.h>
 #include <linux/printk.h>
 #include <linux/build_bug.h>
+#include <asm-generic/irq_pipeline.h>
 #include <asm/byteorder.h>
 #include <asm/div64.h>
 #include <uapi/linux/kernel.h>
@@ -195,9 +196,12 @@ struct user;
 
 #ifdef CONFIG_PREEMPT_VOLUNTARY
 extern int _cond_resched(void);
-# define might_resched() _cond_resched()
+# define might_resched() do { \
+		check_inband_stage(); \
+		_cond_resched(); \
+	} while (0)
 #else
-# define might_resched() do { } while (0)
+# define might_resched() check_inband_stage()
 #endif
 
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
diff --git a/kernel/include/linux/kvm_host.h b/kernel/include/linux/kvm_host.h
index 9cb0a3d787..36d741e2a4 100644
--- a/kernel/include/linux/kvm_host.h
+++ b/kernel/include/linux/kvm_host.h
@@ -14,6 +14,7 @@
 #include <linux/mm.h>
 #include <linux/mmu_notifier.h>
 #include <linux/preempt.h>
+#include <linux/dovetail.h>
 #include <linux/msi.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
@@ -260,10 +261,23 @@ struct kvm_mmio_fragment {
 	unsigned len;
 };
 
+/*
+ * Called when the host is about to leave the inband stage. Typically
+ * used for switching the current vcpu out of guest mode before a
+ * companion core reinstates an oob task context.
+ */
+struct kvm_oob_notifier {
+	void (*handler)(struct kvm_oob_notifier *nfy);
+	bool put_vcpu;
+};
+
 struct kvm_vcpu {
 	struct kvm *kvm;
 #ifdef CONFIG_PREEMPT_NOTIFIERS
 	struct preempt_notifier preempt_notifier;
+#endif
+#ifdef CONFIG_DOVETAIL
+	struct kvm_oob_notifier oob_notifier;
 #endif
 	int cpu;
 	int vcpu_id; /* id given by userspace at creation */
@@ -1502,6 +1516,47 @@ static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
 }
 #endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */
 
+#if defined(CONFIG_DOVETAIL) && defined(CONFIG_KVM)
+static inline void inband_init_vcpu(struct kvm_vcpu *vcpu,
+		void (*preempt_handler)(struct kvm_oob_notifier *nfy))
+{
+	vcpu->oob_notifier.handler = preempt_handler;
+	vcpu->oob_notifier.put_vcpu = false;
+}
+
+static inline void inband_enter_guest(struct kvm_vcpu *vcpu)
+{
+	struct irq_pipeline_data *p = raw_cpu_ptr(&irq_pipeline);
+	WRITE_ONCE(p->vcpu_notify, &vcpu->oob_notifier);
+}
+
+static inline void inband_exit_guest(void)
+{
+	struct irq_pipeline_data *p = raw_cpu_ptr(&irq_pipeline);
+	WRITE_ONCE(p->vcpu_notify, NULL);
+}
+
+static inline void inband_set_vcpu_release_state(struct kvm_vcpu *vcpu,
+						bool pending)
+{
+	vcpu->oob_notifier.put_vcpu = pending;
+}
+#else
+static inline void inband_init_vcpu(struct kvm_vcpu *vcpu,
+		void (*preempt_handler)(struct kvm_oob_notifier *nfy))
+{ }
+
+static inline void inband_enter_guest(struct kvm_vcpu *vcpu)
+{ }
+
+static inline void inband_exit_guest(void)
+{ }
+
+static inline void inband_set_vcpu_release_state(struct kvm_vcpu *vcpu,
+						bool pending)
+{ }
+#endif
+
 typedef int (*kvm_vm_thread_fn_t)(struct kvm *kvm, uintptr_t data);
 
 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn,
diff --git a/kernel/include/linux/lockdep.h b/kernel/include/linux/lockdep.h
index 2c2586312b..3a6b855e3e 100644
--- a/kernel/include/linux/lockdep.h
+++ b/kernel/include/linux/lockdep.h
@@ -214,29 +214,30 @@ static inline void lockdep_init_map(struct lockdep_map *lock, const char *name,
  * of dependencies wrong: they are either too broad (they need a class-split)
  * or they are too narrow (they suffer from a false class-split):
  */
-#define lockdep_set_class(lock, key)				\
-	lockdep_init_map_type(&(lock)->dep_map, #key, key, 0,	\
-			      (lock)->dep_map.wait_type_inner,	\
-			      (lock)->dep_map.wait_type_outer,	\
-			      (lock)->dep_map.lock_type)
-
-#define lockdep_set_class_and_name(lock, key, name)		\
-	lockdep_init_map_type(&(lock)->dep_map, name, key, 0,	\
-			      (lock)->dep_map.wait_type_inner,	\
-			      (lock)->dep_map.wait_type_outer,	\
-			      (lock)->dep_map.lock_type)
-
-#define lockdep_set_class_and_subclass(lock, key, sub)		\
-	lockdep_init_map_type(&(lock)->dep_map, #key, key, sub,	\
-			      (lock)->dep_map.wait_type_inner,	\
-			      (lock)->dep_map.wait_type_outer,	\
-			      (lock)->dep_map.lock_type)
+#define lockdep_set_class(lock, key)					\
+	lockdep_init_map_type(LOCKDEP_ALT_DEPMAP(lock), #key, key, 0,	\
+			LOCKDEP_ALT_DEPMAP(lock)->wait_type_inner,	\
+			LOCKDEP_ALT_DEPMAP(lock)->wait_type_outer,	\
+			LOCKDEP_ALT_DEPMAP(lock)->lock_type)
+
+#define lockdep_set_class_and_name(lock, key, name)			\
+	lockdep_init_map_type(LOCKDEP_ALT_DEPMAP(lock), name, key, 0,	\
+			LOCKDEP_ALT_DEPMAP(lock)->wait_type_inner,	\
+			LOCKDEP_ALT_DEPMAP(lock)->wait_type_outer,	\
+			LOCKDEP_ALT_DEPMAP(lock)->lock_type)
+
+#define lockdep_set_class_and_subclass(lock, key, sub)			\
+	lockdep_init_map_type(LOCKDEP_ALT_DEPMAP(lock), #key, key, sub,	\
+			LOCKDEP_ALT_DEPMAP(lock)->wait_type_inner,	\
+			LOCKDEP_ALT_DEPMAP(lock)->wait_type_outer,	\
+			LOCKDEP_ALT_DEPMAP(lock)->lock_type)
 
 #define lockdep_set_subclass(lock, sub)					\
-	lockdep_init_map_type(&(lock)->dep_map, #lock, (lock)->dep_map.key, sub,\
-			      (lock)->dep_map.wait_type_inner,		\
-			      (lock)->dep_map.wait_type_outer,		\
-			      (lock)->dep_map.lock_type)
+	lockdep_init_map_type(LOCKDEP_ALT_DEPMAP(lock), #lock,		\
+			LOCKDEP_ALT_DEPMAP(lock)->key, sub,		\
+			LOCKDEP_ALT_DEPMAP(lock)->wait_type_inner,	\
+			LOCKDEP_ALT_DEPMAP(lock)->wait_type_outer,	\
+			LOCKDEP_ALT_DEPMAP(lock)->lock_type)
 
 #define lockdep_set_novalidate_class(lock) \
 	lockdep_set_class_and_name(lock, &__lockdep_no_validate__, #lock)
@@ -244,7 +245,8 @@ static inline void lockdep_init_map(struct lockdep_map *lock, const char *name,
 /*
  * Compare locking classes
  */
-#define lockdep_match_class(lock, key) lockdep_match_key(&(lock)->dep_map, key)
+#define lockdep_match_class(lock, key) \
+	lockdep_match_key(LOCKDEP_ALT_DEPMAP(lock), key)
 
 static inline int lockdep_match_key(struct lockdep_map *lock,
 				    struct lock_class_key *key)
@@ -282,8 +284,8 @@ static inline int lock_is_held(const struct lockdep_map *lock)
 	return lock_is_held_type(lock, -1);
 }
 
-#define lockdep_is_held(lock)		lock_is_held(&(lock)->dep_map)
-#define lockdep_is_held_type(lock, r)	lock_is_held_type(&(lock)->dep_map, (r))
+#define lockdep_is_held(lock)		lock_is_held(LOCKDEP_ALT_DEPMAP(lock))
+#define lockdep_is_held_type(lock, r)	lock_is_held_type(LOCKDEP_ALT_DEPMAP(lock), (r))
 
 extern void lock_set_class(struct lockdep_map *lock, const char *name,
 			   struct lock_class_key *key, unsigned int subclass,
@@ -306,26 +308,27 @@ extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);
 #define lockdep_depth(tsk)	(debug_locks ? (tsk)->lockdep_depth : 0)
 
 #define lockdep_assert_held(l)	do {				\
-		WARN_ON(debug_locks && !lockdep_is_held(l));	\
+		WARN_ON(debug_locks && !LOCKDEP_HARD_DEBUG_RET(l, 1, lockdep_is_held(l))); \
 	} while (0)
 
 #define lockdep_assert_held_write(l)	do {			\
-		WARN_ON(debug_locks && !lockdep_is_held_type(l, 0));	\
+		WARN_ON(debug_locks && !LOCKDEP_HARD_DEBUG_RET(l, 1, lockdep_is_held_type(l, 0))); \
 	} while (0)
 
 #define lockdep_assert_held_read(l)	do {				\
-		WARN_ON(debug_locks && !lockdep_is_held_type(l, 1));	\
+		WARN_ON(debug_locks && !LOCKDEP_HARD_DEBUG_RET(l, 1, lockdep_is_held_type(l, 1))); \
 	} while (0)
 
 #define lockdep_assert_held_once(l)	do {				\
-		WARN_ON_ONCE(debug_locks && !lockdep_is_held(l));	\
+		WARN_ON_ONCE(debug_locks && !LOCKDEP_HARD_DEBUG_RET(l, 1, lockdep_is_held(l))); \
 	} while (0)
 
 #define lockdep_recursing(tsk)	((tsk)->lockdep_recursion)
 
-#define lockdep_pin_lock(l)	lock_pin_lock(&(l)->dep_map)
-#define lockdep_repin_lock(l,c)	lock_repin_lock(&(l)->dep_map, (c))
-#define lockdep_unpin_lock(l,c)	lock_unpin_lock(&(l)->dep_map, (c))
+#define lockdep_pin_lock(l)	LOCKDEP_HARD_DEBUG_RET(l, ({ struct pin_cookie cookie; cookie;} ), \
+							lock_pin_lock(LOCKDEP_ALT_DEPMAP(l)))
+#define lockdep_repin_lock(l,c)	LOCKDEP_HARD_DEBUG(l,, lock_repin_lock(LOCKDEP_ALT_DEPMAP(l), (c)))
+#define lockdep_unpin_lock(l,c)	LOCKDEP_HARD_DEBUG(l,, lock_unpin_lock(LOCKDEP_ALT_DEPMAP(l), (c)))
 
 #else /* !CONFIG_LOCKDEP */
 
@@ -552,22 +555,22 @@ do {									\
 #ifdef CONFIG_PROVE_LOCKING
 # define might_lock(lock)						\
 do {									\
-	typecheck(struct lockdep_map *, &(lock)->dep_map);		\
-	lock_acquire(&(lock)->dep_map, 0, 0, 0, 1, NULL, _THIS_IP_);	\
-	lock_release(&(lock)->dep_map, _THIS_IP_);			\
+	typecheck(struct lockdep_map *, LOCKDEP_ALT_DEPMAP(lock));	\
+	lock_acquire(LOCKDEP_ALT_DEPMAP(lock), 0, 0, 0, 1, NULL, _THIS_IP_);	\
+	lock_release(LOCKDEP_ALT_DEPMAP(lock), _THIS_IP_);			\
 } while (0)
 # define might_lock_read(lock)						\
 do {									\
-	typecheck(struct lockdep_map *, &(lock)->dep_map);		\
-	lock_acquire(&(lock)->dep_map, 0, 0, 1, 1, NULL, _THIS_IP_);	\
-	lock_release(&(lock)->dep_map, _THIS_IP_);			\
+	typecheck(struct lockdep_map *, LOCKDEP_ALT_DEPMAP(lock));	\
+	lock_acquire(LOCKDEP_ALT_DEPMAP(lock), 0, 0, 1, 1, NULL, _THIS_IP_); \
+	lock_release(LOCKDEP_ALT_DEPMAP(lock), _THIS_IP_);		\
 } while (0)
 # define might_lock_nested(lock, subclass)				\
 do {									\
-	typecheck(struct lockdep_map *, &(lock)->dep_map);		\
-	lock_acquire(&(lock)->dep_map, subclass, 0, 1, 1, NULL,		\
+	typecheck(struct lockdep_map *, LOCKDEP_ALT_DEPMAP(lock));	\
+	lock_acquire(LOCKDEP_ALT_DEPMAP(lock), subclass, 0, 1, 1, NULL,	\
 		     _THIS_IP_);					\
-	lock_release(&(lock)->dep_map, _THIS_IP_);			\
+	lock_release(LOCKDEP_ALT_DEPMAP(lock), _THIS_IP_);		\
 } while (0)
 
 DECLARE_PER_CPU(int, hardirqs_enabled);
@@ -576,14 +579,32 @@ DECLARE_PER_CPU(unsigned int, lockdep_recursion);
 
 #define __lockdep_enabled	(debug_locks && !this_cpu_read(lockdep_recursion))
 
+#define __lockdep_check_irqs_enabled()					\
+	({ !hard_irqs_disabled() &&					\
+		(running_oob() || this_cpu_read(hardirqs_enabled)); })
+
 #define lockdep_assert_irqs_enabled()					\
-do {									\
-	WARN_ON_ONCE(__lockdep_enabled && !this_cpu_read(hardirqs_enabled)); \
-} while (0)
+	do {								\
+		WARN_ON_ONCE(__lockdep_enabled &&			\
+			!__lockdep_check_irqs_enabled());		\
+	} while (0)
+
+#define __lockdep_check_irqs_disabled()					\
+	({ hard_irqs_disabled() ||					\
+		(running_inband() && !this_cpu_read(hardirqs_enabled)); })
 
 #define lockdep_assert_irqs_disabled()					\
+	  do {								\
+		  WARN_ON_ONCE(__lockdep_enabled &&			\
+			  !__lockdep_check_irqs_disabled());		\
+	  } while (0)
+
+#define lockdep_read_irqs_state()					\
+	({ this_cpu_read(hardirqs_enabled); })
+
+#define lockdep_write_irqs_state(__state)				\
 do {									\
-	WARN_ON_ONCE(__lockdep_enabled && this_cpu_read(hardirqs_enabled)); \
+	this_cpu_write(hardirqs_enabled, __state);			\
 } while (0)
 
 #define lockdep_assert_in_irq()						\
@@ -596,7 +617,7 @@ do {									\
 	WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT)	&&		\
 		     __lockdep_enabled			&&		\
 		     (preempt_count() != 0		||		\
-		      !this_cpu_read(hardirqs_enabled)));		\
+		     __lockdep_check_irqs_disabled()));			\
 } while (0)
 
 #define lockdep_assert_preemption_disabled()				\
@@ -604,7 +625,7 @@ do {									\
 	WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT)	&&		\
 		     __lockdep_enabled			&&		\
 		     (preempt_count() == 0		&&		\
-		      this_cpu_read(hardirqs_enabled)));		\
+		     __lockdep_check_irqs_enabled()));			\
 } while (0)
 
 #else
@@ -614,6 +635,8 @@ do {									\
 
 # define lockdep_assert_irqs_enabled() do { } while (0)
 # define lockdep_assert_irqs_disabled() do { } while (0)
+# define lockdep_read_irqs_state() 0
+# define lockdep_write_irqs_state(__state) do { (void)(__state); } while (0)
 # define lockdep_assert_in_irq() do { } while (0)
 
 # define lockdep_assert_preemption_enabled() do { } while (0)
diff --git a/kernel/include/linux/mm.h b/kernel/include/linux/mm.h
index dfefcfa1d6..7e23752bdc 100644
--- a/kernel/include/linux/mm.h
+++ b/kernel/include/linux/mm.h
@@ -20,6 +20,7 @@
 #include <linux/pfn.h>
 #include <linux/percpu-refcount.h>
 #include <linux/bit_spinlock.h>
+#include <linux/dovetail.h>
 #include <linux/shrinker.h>
 #include <linux/resource.h>
 #include <linux/page_ext.h>
diff --git a/kernel/include/linux/mm_types.h b/kernel/include/linux/mm_types.h
index c853f612a8..5820ea308a 100644
--- a/kernel/include/linux/mm_types.h
+++ b/kernel/include/linux/mm_types.h
@@ -19,6 +19,8 @@
 
 #include <asm/mmu.h>
 
+#include <dovetail/mm_info.h>
+
 #ifndef AT_VECTOR_SIZE_ARCH
 #define AT_VECTOR_SIZE_ARCH 0
 #endif
@@ -592,6 +594,9 @@ struct mm_struct {
 		struct uprobes_state uprobes_state;
 #ifdef CONFIG_HUGETLB_PAGE
 		atomic_long_t hugetlb_usage;
+#endif
+#ifdef CONFIG_DOVETAIL
+		struct oob_mm_state oob_state;
 #endif
 		struct work_struct async_put_work;
 
diff --git a/kernel/include/linux/net.h b/kernel/include/linux/net.h
index e201a7fbf3..f963695bb8 100644
--- a/kernel/include/linux/net.h
+++ b/kernel/include/linux/net.h
@@ -79,6 +79,7 @@ enum sock_type {
 #ifndef SOCK_NONBLOCK
 #define SOCK_NONBLOCK	O_NONBLOCK
 #endif
+#define SOCK_OOB	O_OOB
 
 #endif /* ARCH_HAS_SOCKET_TYPES */
 
diff --git a/kernel/include/linux/netdevice.h b/kernel/include/linux/netdevice.h
index 7e7a003dd2..d25e9aa0f3 100644
--- a/kernel/include/linux/netdevice.h
+++ b/kernel/include/linux/netdevice.h
@@ -41,6 +41,7 @@
 #endif
 #include <net/netprio_cgroup.h>
 #include <net/xdp.h>
+#include <net/netoob.h>
 
 #include <linux/netdev_features.h>
 #include <linux/neighbour.h>
@@ -296,6 +297,7 @@ enum netdev_state_t {
 	__LINK_STATE_LINKWATCH_PENDING,
 	__LINK_STATE_DORMANT,
 	__LINK_STATE_TESTING,
+	__LINK_STATE_OOB,
 };
 
 
@@ -1534,6 +1536,13 @@ struct net_device_ops {
 	ANDROID_KABI_RESERVE(6);
 	ANDROID_KABI_RESERVE(7);
 	ANDROID_KABI_RESERVE(8);
+#ifdef CONFIG_NET_OOB
+	struct sk_buff *	(*ndo_alloc_oob_skb)(struct net_device *dev,
+						     dma_addr_t *dma_addr);
+	void			(*ndo_free_oob_skb)(struct net_device *dev,
+						    struct sk_buff *skb,
+						    dma_addr_t dma_addr);
+#endif
 };
 
 /**
@@ -1725,6 +1734,7 @@ enum netdev_ml_priv_type {
  *	@tlsdev_ops:	Transport Layer Security offload operations
  *	@header_ops:	Includes callbacks for creating,parsing,caching,etc
  *			of Layer 2 headers.
+ *	@net_oob_context:	Out-of-band networking context (oob stage diversion)
  *
  *	@flags:		Interface flags (a la BSD)
  *	@priv_flags:	Like 'flags' but invisible to userspace,
@@ -1984,6 +1994,10 @@ struct net_device {
 	const struct tlsdev_ops *tlsdev_ops;
 #endif
 
+#ifdef CONFIG_NET_OOB
+	struct oob_netdev_context  oob_context;
+#endif
+
 	const struct header_ops *header_ops;
 
 	unsigned int		flags;
@@ -4190,6 +4204,86 @@ void netif_device_detach(struct net_device *dev);
 
 void netif_device_attach(struct net_device *dev);
 
+#ifdef CONFIG_NET_OOB
+
+static inline bool netif_oob_diversion(const struct net_device *dev)
+{
+	return test_bit(__LINK_STATE_OOB, &dev->state);
+}
+
+static inline void netif_enable_oob_diversion(struct net_device *dev)
+{
+	return set_bit(__LINK_STATE_OOB, &dev->state);
+}
+
+static inline void netif_disable_oob_diversion(struct net_device *dev)
+{
+	clear_bit(__LINK_STATE_OOB, &dev->state);
+	smp_mb__after_atomic();
+}
+
+int netif_xmit_oob(struct sk_buff *skb);
+
+static inline bool netdev_is_oob_capable(struct net_device *dev)
+{
+	return !!(dev->oob_context.flags & IFF_OOB_CAPABLE);
+}
+
+static inline void netdev_enable_oob_port(struct net_device *dev)
+{
+	dev->oob_context.flags |= IFF_OOB_PORT;
+}
+
+static inline void netdev_disable_oob_port(struct net_device *dev)
+{
+	dev->oob_context.flags &= ~IFF_OOB_PORT;
+}
+
+static inline bool netdev_is_oob_port(struct net_device *dev)
+{
+	return !!(dev->oob_context.flags & IFF_OOB_PORT);
+}
+
+static inline struct sk_buff *netdev_alloc_oob_skb(struct net_device *dev,
+						   dma_addr_t *dma_addr)
+{
+	return dev->netdev_ops->ndo_alloc_oob_skb(dev, dma_addr);
+}
+
+static inline void netdev_free_oob_skb(struct net_device *dev,
+				       struct sk_buff *skb,
+				       dma_addr_t dma_addr)
+{
+	dev->netdev_ops->ndo_free_oob_skb(dev, skb, dma_addr);
+}
+
+#else
+
+static inline bool netif_oob_diversion(const struct net_device *dev)
+{
+	return false;
+}
+
+static inline bool netdev_is_oob_capable(struct net_device *dev)
+{
+	return false;
+}
+
+static inline void netdev_enable_oob_port(struct net_device *dev)
+{
+}
+
+static inline void netdev_disable_oob_port(struct net_device *dev)
+{
+}
+
+static inline bool netdev_is_oob_port(struct net_device *dev)
+{
+	return false;
+}
+
+#endif
+
 /*
  * Network interface message level settings
  */
diff --git a/kernel/include/linux/poll.h b/kernel/include/linux/poll.h
index 1cdc32b1f1..2701db20ac 100644
--- a/kernel/include/linux/poll.h
+++ b/kernel/include/linux/poll.h
@@ -10,6 +10,7 @@
 #include <linux/fs.h>
 #include <linux/sysctl.h>
 #include <linux/uaccess.h>
+#include <dovetail/poll.h>
 #include <uapi/linux/poll.h>
 #include <uapi/linux/eventpoll.h>
 
diff --git a/kernel/include/linux/preempt.h b/kernel/include/linux/preempt.h
index 7d9c1c0e14..58c21bc461 100644
--- a/kernel/include/linux/preempt.h
+++ b/kernel/include/linux/preempt.h
@@ -27,17 +27,23 @@
  *         SOFTIRQ_MASK:	0x0000ff00
  *         HARDIRQ_MASK:	0x000f0000
  *             NMI_MASK:	0x00f00000
+ *         PIPELINE_MASK:	0x01000000
+ *         STAGE_MASK:		0x02000000
  * PREEMPT_NEED_RESCHED:	0x80000000
  */
 #define PREEMPT_BITS	8
 #define SOFTIRQ_BITS	8
 #define HARDIRQ_BITS	4
 #define NMI_BITS	4
+#define PIPELINE_BITS	1
+#define STAGE_BITS	1
 
 #define PREEMPT_SHIFT	0
 #define SOFTIRQ_SHIFT	(PREEMPT_SHIFT + PREEMPT_BITS)
 #define HARDIRQ_SHIFT	(SOFTIRQ_SHIFT + SOFTIRQ_BITS)
 #define NMI_SHIFT	(HARDIRQ_SHIFT + HARDIRQ_BITS)
+#define PIPELINE_SHIFT	(NMI_SHIFT + NMI_BITS)
+#define STAGE_SHIFT	(PIPELINE_SHIFT + PIPELINE_BITS)
 
 #define __IRQ_MASK(x)	((1UL << (x))-1)
 
@@ -45,11 +51,15 @@
 #define SOFTIRQ_MASK	(__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
 #define HARDIRQ_MASK	(__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
 #define NMI_MASK	(__IRQ_MASK(NMI_BITS)     << NMI_SHIFT)
+#define PIPELINE_MASK	(__IRQ_MASK(PIPELINE_BITS) << PIPELINE_SHIFT)
+#define STAGE_MASK	(__IRQ_MASK(STAGE_BITS) << STAGE_SHIFT)
 
 #define PREEMPT_OFFSET	(1UL << PREEMPT_SHIFT)
 #define SOFTIRQ_OFFSET	(1UL << SOFTIRQ_SHIFT)
 #define HARDIRQ_OFFSET	(1UL << HARDIRQ_SHIFT)
 #define NMI_OFFSET	(1UL << NMI_SHIFT)
+#define PIPELINE_OFFSET	(1UL << PIPELINE_SHIFT)
+#define STAGE_OFFSET	(1UL << STAGE_SHIFT)
 
 #define SOFTIRQ_DISABLE_OFFSET	(2 * SOFTIRQ_OFFSET)
 
@@ -82,6 +92,9 @@
 #define irq_count()	(preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
 				 | NMI_MASK))
 
+/* The current IRQ stage level: 0=inband, 1=oob */
+#define stage_level()	((preempt_count() & STAGE_MASK) >> STAGE_SHIFT)
+
 /*
  * Are we doing bottom half or hardware interrupt processing?
  *
@@ -91,6 +104,7 @@
  * in_serving_softirq() - We're in softirq context
  * in_nmi()       - We're in NMI context
  * in_task()	  - We're in task context
+ * in_pipeline()  - We're on pipeline entry
  *
  * Note: due to the BH disabled confusion: in_softirq(),in_interrupt() really
  *       should not be used in new code.
@@ -102,6 +116,7 @@
 #define in_nmi()		(preempt_count() & NMI_MASK)
 #define in_task()		(!(preempt_count() & \
 				   (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
+#define in_pipeline()		(preempt_count() & PIPELINE_MASK)
 
 /*
  * The preempt_count offset after preempt_disable();
@@ -180,7 +195,8 @@ do { \
 
 #define preempt_enable_no_resched() sched_preempt_enable_no_resched()
 
-#define preemptible()	(preempt_count() == 0 && !irqs_disabled())
+#define preemptible()	(preempt_count() == 0 && \
+			 !hard_irqs_disabled() && !irqs_disabled())
 
 #ifdef CONFIG_PREEMPTION
 #define preempt_enable() \
@@ -352,4 +368,43 @@ static __always_inline void migrate_enable(void)
 	preempt_enable();
 }
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+static __always_inline bool running_inband(void)
+{
+	return stage_level() == 0;
+}
+
+static __always_inline bool running_oob(void)
+{
+	return !running_inband();
+}
+
+unsigned long hard_preempt_disable(void);
+void hard_preempt_enable(unsigned long flags);
+
+#else
+
+static __always_inline bool running_inband(void)
+{
+	return true;
+}
+
+static __always_inline bool running_oob(void)
+{
+	return false;
+}
+
+#define hard_preempt_disable()		\
+({					\
+	preempt_disable();		\
+	0;				\
+})
+#define hard_preempt_enable(__flags)	\
+	do {				\
+		preempt_enable();	\
+		(void)(__flags);	\
+	} while (0)
+#endif
+
 #endif /* __LINUX_PREEMPT_H */
diff --git a/kernel/include/linux/printk.h b/kernel/include/linux/printk.h
index 14d13ecaa8..270fa48ce0 100644
--- a/kernel/include/linux/printk.h
+++ b/kernel/include/linux/printk.h
@@ -160,8 +160,23 @@ static inline void printk_nmi_direct_enter(void) { }
 static inline void printk_nmi_direct_exit(void) { }
 #endif /* PRINTK_NMI */
 
+
 struct dev_printk_info;
 
+#ifdef CONFIG_RAW_PRINTK
+void raw_puts(const char *s, size_t len);
+void raw_vprintk(const char *fmt, va_list ap);
+asmlinkage __printf(1, 2)
+void raw_printk(const char *fmt, ...);
+#else
+static inline __cold
+void raw_puts(const char *s, size_t len) { }
+static inline __cold
+void raw_vprintk(const char *s, va_list ap) { }
+static inline __printf(1, 2) __cold
+void raw_printk(const char *s, ...) { }
+#endif
+
 #ifdef CONFIG_PRINTK
 asmlinkage __printf(4, 0)
 int vprintk_emit(int facility, int level,
@@ -512,7 +527,7 @@ extern int kptr_restrict;
 				      DEFAULT_RATELIMIT_INTERVAL,	\
 				      DEFAULT_RATELIMIT_BURST);		\
 									\
-	if (__ratelimit(&_rs))						\
+	if (running_oob() || __ratelimit(&_rs))				\
 		printk(fmt, ##__VA_ARGS__);				\
 })
 #else
diff --git a/kernel/include/linux/rcupdate.h b/kernel/include/linux/rcupdate.h
index 095b3b39bd..a4388efe67 100644
--- a/kernel/include/linux/rcupdate.h
+++ b/kernel/include/linux/rcupdate.h
@@ -118,6 +118,14 @@ static inline void rcu_init_nohz(void) { }
 static inline void rcu_nocb_flush_deferred_wakeup(void) { }
 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
 
+#ifdef CONFIG_IRQ_PIPELINE
+void rcu_oob_prepare_lock(void);
+void rcu_oob_finish_lock(void);
+#else
+#define rcu_oob_prepare_lock()	 do { } while (0)
+#define rcu_oob_finish_lock()	 do { } while (0)
+#endif
+
 /**
  * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
  * @a: Code that RCU needs to pay attention to.
@@ -290,7 +298,7 @@ static inline int rcu_read_lock_bh_held(void)
 
 static inline int rcu_read_lock_sched_held(void)
 {
-	return !preemptible();
+	return !running_inband() || !preemptible();
 }
 
 static inline int rcu_read_lock_any_held(void)
@@ -646,6 +654,7 @@ do {									      \
  */
 static __always_inline void rcu_read_lock(void)
 {
+	rcu_oob_prepare_lock();
 	__rcu_read_lock();
 	__acquire(RCU);
 	rcu_lock_acquire(&rcu_lock_map);
@@ -702,6 +711,7 @@ static inline void rcu_read_unlock(void)
 			 "rcu_read_unlock() used illegally while idle");
 	__release(RCU);
 	__rcu_read_unlock();
+	rcu_oob_finish_lock();
 	rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */
 }
 
@@ -755,6 +765,7 @@ static inline void rcu_read_unlock_bh(void)
 static inline void rcu_read_lock_sched(void)
 {
 	preempt_disable();
+	rcu_oob_prepare_lock();
 	__acquire(RCU_SCHED);
 	rcu_lock_acquire(&rcu_sched_lock_map);
 	RCU_LOCKDEP_WARN(!rcu_is_watching(),
@@ -779,6 +790,7 @@ static inline void rcu_read_unlock_sched(void)
 			 "rcu_read_unlock_sched() used illegally while idle");
 	rcu_lock_release(&rcu_sched_lock_map);
 	__release(RCU_SCHED);
+	rcu_oob_finish_lock();
 	preempt_enable();
 }
 
diff --git a/kernel/include/linux/regmap.h b/kernel/include/linux/regmap.h
index 751ca38693..4df7952493 100644
--- a/kernel/include/linux/regmap.h
+++ b/kernel/include/linux/regmap.h
@@ -369,6 +369,7 @@ struct regmap_config {
 	int (*reg_write)(void *context, unsigned int reg, unsigned int val);
 
 	bool fast_io;
+	bool oob_io;
 
 	unsigned int max_register;
 	const struct regmap_access_table *wr_table;
diff --git a/kernel/include/linux/sched.h b/kernel/include/linux/sched.h
index d3cc279a46..38cac8c3e5 100644
--- a/kernel/include/linux/sched.h
+++ b/kernel/include/linux/sched.h
@@ -119,6 +119,12 @@ struct io_uring_task;
 
 #define task_is_stopped_or_traced(task)	((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
 
+#ifdef CONFIG_DOVETAIL
+#define task_is_off_stage(task)		test_ti_local_flags(task_thread_info(task), _TLF_OFFSTAGE)
+#else
+#define task_is_off_stage(task)		0
+#endif
+
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
 
 /*
@@ -1052,6 +1058,10 @@ struct task_struct {
 	int				irq_config;
 #endif
 
+#ifdef CONFIG_IRQ_PIPELINE
+	unsigned long			stall_bits;
+#endif
+
 #ifdef CONFIG_LOCKDEP
 # define MAX_LOCK_DEPTH			48UL
 	u64				curr_chain_key;
diff --git a/kernel/include/linux/sched/coredump.h b/kernel/include/linux/sched/coredump.h
index dfd82eab29..0b06940606 100644
--- a/kernel/include/linux/sched/coredump.h
+++ b/kernel/include/linux/sched/coredump.h
@@ -74,6 +74,7 @@ static inline int get_dumpable(struct mm_struct *mm)
 #define MMF_OOM_REAP_QUEUED	26	/* mm was queued for oom_reaper */
 #define MMF_MULTIPROCESS	27	/* mm is shared between processes */
 #define MMF_DISABLE_THP_MASK	(1 << MMF_DISABLE_THP)
+#define MMF_DOVETAILED		31	/* mm belongs to a dovetailed process */
 
 #define MMF_INIT_MASK		(MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\
 				 MMF_DISABLE_THP_MASK)
diff --git a/kernel/include/linux/skbuff.h b/kernel/include/linux/skbuff.h
index f73efb3057..3556cf6e03 100644
--- a/kernel/include/linux/skbuff.h
+++ b/kernel/include/linux/skbuff.h
@@ -793,6 +793,12 @@ struct sk_buff {
 #ifdef CONFIG_SKB_EXTENSIONS
 	__u8			active_extensions;
 #endif
+#ifdef CONFIG_NET_OOB
+	__u8			oob:1;
+	__u8			oob_clone:1;
+	__u8			oob_cloned:1;
+#endif
+
 	/* fields enclosed in headers_start/headers_end are copied
 	 * using a single memcpy() in __copy_skb_header()
 	 */
@@ -1102,6 +1108,69 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size);
 struct sk_buff *build_skb(void *data, unsigned int frag_size);
 struct sk_buff *build_skb_around(struct sk_buff *skb,
 				 void *data, unsigned int frag_size);
+#ifdef CONFIG_NET_OOB
+
+static inline void __skb_oob_copy(struct sk_buff *new,
+				const struct sk_buff *old)
+{
+	new->oob = old->oob;
+	new->oob_clone = old->oob_clone;
+	new->oob_cloned = old->oob_cloned;
+}
+
+static inline bool skb_is_oob(const struct sk_buff *skb)
+{
+	return skb->oob;
+}
+
+static inline bool skb_is_oob_clone(const struct sk_buff *skb)
+{
+	return skb->oob_clone;
+}
+
+static inline bool skb_has_oob_clone(const struct sk_buff *skb)
+{
+	return skb->oob_cloned;
+}
+
+struct sk_buff *__netdev_alloc_oob_skb(struct net_device *dev,
+				size_t len, size_t headroom,
+				gfp_t gfp_mask);
+void __netdev_free_oob_skb(struct net_device *dev, struct sk_buff *skb);
+void netdev_reset_oob_skb(struct net_device *dev, struct sk_buff *skb,
+			size_t headroom);
+struct sk_buff *skb_alloc_oob_head(gfp_t gfp_mask);
+void skb_morph_oob_skb(struct sk_buff *n, struct sk_buff *skb);
+bool skb_release_oob_skb(struct sk_buff *skb, int *dref);
+
+static inline bool recycle_oob_skb(struct sk_buff *skb)
+{
+	bool skb_oob_recycle(struct sk_buff *skb);
+
+	if (!skb->oob)
+		return false;
+
+	return skb_oob_recycle(skb);
+}
+
+#else  /* !CONFIG_NET_OOB */
+
+static inline void __skb_oob_copy(struct sk_buff *new,
+				const struct sk_buff *old)
+{
+}
+
+static inline bool skb_is_oob(const struct sk_buff *skb)
+{
+	return false;
+}
+
+static inline bool recycle_oob_skb(struct sk_buff *skb)
+{
+	return false;
+}
+
+#endif	/* !CONFIG_NET_OOB */
 
 /**
  * alloc_skb - allocate a network buffer
diff --git a/kernel/include/linux/smp.h b/kernel/include/linux/smp.h
index 7ce15c3be7..44888c0819 100644
--- a/kernel/include/linux/smp.h
+++ b/kernel/include/linux/smp.h
@@ -241,6 +241,21 @@ static inline int get_boot_cpu_id(void)
 #define get_cpu()		({ preempt_disable(); __smp_processor_id(); })
 #define put_cpu()		preempt_enable()
 
+#ifdef CONFIG_IRQ_PIPELINE
+#define hard_get_cpu(flags)	({			\
+		(flags) = hard_preempt_disable();	\
+		raw_smp_processor_id();			\
+	})
+#define hard_put_cpu(flags)	hard_preempt_enable(flags)
+#else
+#define hard_get_cpu(flags)	({ (void)(flags); get_cpu(); })
+#define hard_put_cpu(flags)	\
+	do {			\
+		(void)(flags);	\
+		put_cpu();	\
+	} while (0)
+#endif
+
 /*
  * Callback to arch code if there's nosmp or maxcpus=0 on the
  * boot command line:
diff --git a/kernel/include/linux/socket.h b/kernel/include/linux/socket.h
index c3b35d18bc..eb8bf6aae6 100644
--- a/kernel/include/linux/socket.h
+++ b/kernel/include/linux/socket.h
@@ -223,8 +223,9 @@ struct ucred {
 				 * reuses AF_INET address family
 				 */
 #define AF_XDP		44	/* XDP sockets			*/
+#define AF_OOB		45	/* Out-of-band domain sockets */
 
-#define AF_MAX		45	/* For now.. */
+#define AF_MAX		46	/* For now.. */
 
 /* Protocol families, same as address families. */
 #define PF_UNSPEC	AF_UNSPEC
@@ -274,6 +275,7 @@ struct ucred {
 #define PF_QIPCRTR	AF_QIPCRTR
 #define PF_SMC		AF_SMC
 #define PF_XDP		AF_XDP
+#define PF_OOB		AF_OOB
 #define PF_MAX		AF_MAX
 
 /* Maximum queue length specifiable by listen.  */
diff --git a/kernel/include/linux/spi/spi.h b/kernel/include/linux/spi/spi.h
index f50c766273..04f35c4059 100644
--- a/kernel/include/linux/spi/spi.h
+++ b/kernel/include/linux/spi/spi.h
@@ -9,6 +9,7 @@
 #include <linux/device.h>
 #include <linux/mod_devicetable.h>
 #include <linux/slab.h>
+#include <linux/dmaengine.h>
 #include <linux/kthread.h>
 #include <linux/completion.h>
 #include <linux/scatterlist.h>
@@ -253,6 +254,7 @@ static inline void *spi_get_drvdata(struct spi_device *spi)
 
 struct spi_message;
 struct spi_transfer;
+struct spi_oob_transfer;
 
 /**
  * struct spi_driver - Host side "protocol" driver
@@ -352,6 +354,7 @@ static inline void spi_unregister_driver(struct spi_driver *sdrv)
  * @io_mutex: mutex for physical bus access
  * @bus_lock_spinlock: spinlock for SPI bus locking
  * @bus_lock_mutex: mutex for exclusion of multiple callers
+ * @bus_oob_lock_sem: semaphore for exclusion during oob operations
  * @bus_lock_flag: indicates that the SPI bus is locked for exclusive use
  * @setup: updates the device mode and clocking records used by a
  *	device's SPI controller; protocol code may call this.  This
@@ -534,6 +537,10 @@ struct spi_controller {
 	spinlock_t		bus_lock_spinlock;
 	struct mutex		bus_lock_mutex;
 
+#ifdef CONFIG_SPI_OOB
+	struct semaphore	bus_oob_lock_sem;
+#endif
+
 	/* flag indicating that the SPI bus is locked for exclusive use */
 	bool			bus_lock_flag;
 
@@ -626,6 +633,14 @@ struct spi_controller {
 	int (*unprepare_message)(struct spi_controller *ctlr,
 				 struct spi_message *message);
 	int (*slave_abort)(struct spi_controller *ctlr);
+	int (*prepare_oob_transfer)(struct spi_controller *ctlr,
+				struct spi_oob_transfer *xfer);
+	void (*start_oob_transfer)(struct spi_controller *ctlr,
+				struct spi_oob_transfer *xfer);
+	void (*pulse_oob_transfer)(struct spi_controller *ctlr,
+				struct spi_oob_transfer *xfer);
+	void (*terminate_oob_transfer)(struct spi_controller *ctlr,
+				struct spi_oob_transfer *xfer);
 
 	/*
 	 * These hooks are for drivers that use a generic implementation
@@ -1137,6 +1152,90 @@ static inline void spi_message_free(struct spi_message *m)
 	kfree(m);
 }
 
+struct spi_oob_transfer {
+	struct spi_device *spi;
+	dma_addr_t dma_addr;
+	size_t aligned_frame_len;
+	void *io_buffer;	/* 2 x aligned_frame_len */
+	struct dma_async_tx_descriptor *txd;
+	struct dma_async_tx_descriptor *rxd;
+	u32 effective_speed_hz;
+	/*
+	 * Caller-defined settings for the transfer.
+	 */
+	struct spi_oob_setup {
+		u32 frame_len;
+		u32 speed_hz;
+		u8 bits_per_word;
+		dma_async_tx_callback xfer_done;
+	} setup;
+};
+
+static inline off_t spi_get_oob_rxoff(struct spi_oob_transfer *xfer)
+{
+	/* RX area is in first half of the I/O buffer. */
+	return 0;
+}
+
+static inline off_t spi_get_oob_txoff(struct spi_oob_transfer *xfer)
+{
+	/* TX area is in second half of the I/O buffer. */
+	return xfer->aligned_frame_len;
+}
+
+static inline size_t spi_get_oob_iolen(struct spi_oob_transfer *xfer)
+{
+	return xfer->aligned_frame_len * 2;
+}
+
+#ifdef CONFIG_SPI_OOB
+
+struct vm_area_struct;
+
+int spi_prepare_oob_transfer(struct spi_device *spi,
+			struct spi_oob_transfer *xfer);
+
+void spi_start_oob_transfer(struct spi_oob_transfer *xfer);
+
+int spi_pulse_oob_transfer(struct spi_oob_transfer *xfer);
+
+void spi_terminate_oob_transfer(struct spi_oob_transfer *xfer);
+
+int spi_mmap_oob_transfer(struct vm_area_struct *vma,
+			struct spi_oob_transfer *xfer);
+
+#else
+
+static inline
+int spi_prepare_oob_transfer(struct spi_device *spi,
+			struct spi_oob_transfer *xfer)
+{
+	return -ENOTSUPP;
+}
+
+static inline
+void spi_start_oob_transfer(struct spi_oob_transfer *xfer)
+{ }
+
+static inline
+int spi_pulse_oob_transfer(struct spi_oob_transfer *xfer)
+{
+	return -EIO;
+}
+
+static inline
+void spi_terminate_oob_transfer(struct spi_oob_transfer *xfer)
+{ }
+
+static inline
+int spi_mmap_oob_transfer(struct vm_area_struct *vma,
+			struct spi_oob_transfer *xfer)
+{
+	return -ENXIO;
+}
+
+#endif
+
 extern int spi_set_cs_timing(struct spi_device *spi,
 			     struct spi_delay *setup,
 			     struct spi_delay *hold,
diff --git a/kernel/include/linux/spinlock.h b/kernel/include/linux/spinlock.h
index 79897841a2..311854fe4c 100644
--- a/kernel/include/linux/spinlock.h
+++ b/kernel/include/linux/spinlock.h
@@ -97,21 +97,27 @@
 				   struct lock_class_key *key, short inner);
 
 # define raw_spin_lock_init(lock)					\
+	LOCK_ALTERNATIVES(lock,	spin_lock_init,				\
 do {									\
 	static struct lock_class_key __key;				\
 									\
-	__raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN);	\
-} while (0)
+	__raw_spin_lock_init(__RAWLOCK(lock), #lock, &__key, LD_WAIT_SPIN); \
+} while (0))
 
 #else
 # define raw_spin_lock_init(lock)				\
-	do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
+	LOCK_ALTERNATIVES(lock,	spin_lock_init,			\
+	do { *(__RAWLOCK(lock)) = __RAW_SPIN_LOCK_UNLOCKED(__RAWLOCK(lock)); } while (0))
 #endif
 
-#define raw_spin_is_locked(lock)	arch_spin_is_locked(&(lock)->raw_lock)
+#define raw_spin_is_locked(lock)		\
+	LOCK_ALTERNATIVES_RET(lock, spin_is_locked,	\
+	      arch_spin_is_locked(&(__RAWLOCK(lock))->raw_lock))
 
 #ifdef arch_spin_is_contended
-#define raw_spin_is_contended(lock)	arch_spin_is_contended(&(lock)->raw_lock)
+#define raw_spin_is_contended(lock)			\
+	LOCK_ALTERNATIVES_RET(lock, spin_is_contended,	\
+	      arch_spin_is_contended(&(__RAWLOCK(lock))->raw_lock))
 #else
 #define raw_spin_is_contended(lock)	(((void)(lock), 0))
 #endif /*arch_spin_is_contended*/
@@ -220,13 +226,19 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
  * various methods are defined as nops in the case they are not
  * required.
  */
-#define raw_spin_trylock(lock)	__cond_lock(lock, _raw_spin_trylock(lock))
+#define raw_spin_trylock(lock)			\
+	__cond_lock(lock,			\
+		    LOCK_ALTERNATIVES_RET(lock,	\
+		    spin_trylock, _raw_spin_trylock(__RAWLOCK(lock))))
 
-#define raw_spin_lock(lock)	_raw_spin_lock(lock)
+#define raw_spin_lock(lock)	\
+	LOCK_ALTERNATIVES(lock, spin_lock, _raw_spin_lock(__RAWLOCK(lock)))
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
+
 # define raw_spin_lock_nested(lock, subclass) \
-	_raw_spin_lock_nested(lock, subclass)
+	LOCK_ALTERNATIVES(lock, spin_lock_nested, \
+		_raw_spin_lock_nested(__RAWLOCK(lock), subclass), subclass)
 
 # define raw_spin_lock_nest_lock(lock, nest_lock)			\
 	 do {								\
@@ -239,18 +251,20 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
  * warns about set-but-not-used variables when building with
  * CONFIG_DEBUG_LOCK_ALLOC=n and with W=1.
  */
-# define raw_spin_lock_nested(lock, subclass)		\
-	_raw_spin_lock(((void)(subclass), (lock)))
+# define raw_spin_lock_nested(lock, subclass)	\
+	LOCK_ALTERNATIVES(lock, spin_lock_nested, \
+		_raw_spin_lock(((void)(subclass), __RAWLOCK(lock))), subclass)
 # define raw_spin_lock_nest_lock(lock, nest_lock)	_raw_spin_lock(lock)
 #endif
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 
-#define raw_spin_lock_irqsave(lock, flags)			\
-	do {						\
-		typecheck(unsigned long, flags);	\
-		flags = _raw_spin_lock_irqsave(lock);	\
-	} while (0)
+#define raw_spin_lock_irqsave(lock, flags)				\
+	LOCK_ALTERNATIVES(lock, spin_lock_irqsave,			\
+	do {								\
+		typecheck(unsigned long, flags);			\
+		flags = _raw_spin_lock_irqsave(__RAWLOCK(lock));	\
+	} while (0), flags)
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 #define raw_spin_lock_irqsave_nested(lock, flags, subclass)		\
@@ -268,45 +282,55 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 
 #else
 
-#define raw_spin_lock_irqsave(lock, flags)		\
-	do {						\
-		typecheck(unsigned long, flags);	\
-		_raw_spin_lock_irqsave(lock, flags);	\
-	} while (0)
+#define raw_spin_lock_irqsave(lock, flags)			\
+	LOCK_ALTERNATIVES(lock, spin_lock_irqsave,		\
+	do {							\
+		typecheck(unsigned long, flags);		\
+		_raw_spin_lock_irqsave(__RAWLOCK(lock), flags);	\
+	} while (0), flags)
 
 #define raw_spin_lock_irqsave_nested(lock, flags, subclass)	\
 	raw_spin_lock_irqsave(lock, flags)
 
 #endif
 
-#define raw_spin_lock_irq(lock)		_raw_spin_lock_irq(lock)
+#define raw_spin_lock_irq(lock)		       \
+	LOCK_ALTERNATIVES(lock, spin_lock_irq, \
+			  _raw_spin_lock_irq(__RAWLOCK(lock)))
 #define raw_spin_lock_bh(lock)		_raw_spin_lock_bh(lock)
-#define raw_spin_unlock(lock)		_raw_spin_unlock(lock)
-#define raw_spin_unlock_irq(lock)	_raw_spin_unlock_irq(lock)
-
-#define raw_spin_unlock_irqrestore(lock, flags)		\
-	do {							\
-		typecheck(unsigned long, flags);		\
-		_raw_spin_unlock_irqrestore(lock, flags);	\
-	} while (0)
+#define raw_spin_unlock(lock)		     \
+	LOCK_ALTERNATIVES(lock, spin_unlock, \
+			  _raw_spin_unlock(__RAWLOCK(lock)))
+#define raw_spin_unlock_irq(lock)	\
+	LOCK_ALTERNATIVES(lock, spin_unlock_irq, \
+			  _raw_spin_unlock_irq(__RAWLOCK(lock)))
+
+#define raw_spin_unlock_irqrestore(lock, flags)				\
+	LOCK_ALTERNATIVES(lock, spin_unlock_irqrestore,			\
+	do {								\
+		typecheck(unsigned long, flags);			\
+		_raw_spin_unlock_irqrestore(__RAWLOCK(lock), flags);	\
+	} while (0), flags)
 #define raw_spin_unlock_bh(lock)	_raw_spin_unlock_bh(lock)
 
 #define raw_spin_trylock_bh(lock) \
 	__cond_lock(lock, _raw_spin_trylock_bh(lock))
 
 #define raw_spin_trylock_irq(lock) \
+	LOCK_ALTERNATIVES_RET(lock, spin_trylock_irq, \
 ({ \
 	local_irq_disable(); \
-	raw_spin_trylock(lock) ? \
+	raw_spin_trylock(__RAWLOCK(lock)) ?	\
 	1 : ({ local_irq_enable(); 0;  }); \
-})
+}))
 
 #define raw_spin_trylock_irqsave(lock, flags) \
+	LOCK_ALTERNATIVES_RET(lock, spin_trylock_irqsave, \
 ({ \
 	local_irq_save(flags); \
-	raw_spin_trylock(lock) ? \
+	raw_spin_trylock(__RAWLOCK(lock)) ?	\
 	1 : ({ local_irq_restore(flags); 0; }); \
-})
+}), flags)
 
 /* Include rwlock functions */
 #include <linux/rwlock.h>
@@ -320,12 +344,20 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
 # include <linux/spinlock_api_up.h>
 #endif
 
+/* Pull the lock types specific to the IRQ pipeline. */
+#ifdef CONFIG_IRQ_PIPELINE
+#include <linux/spinlock_pipeline.h>
+#else
+static inline void check_spinlock_context(void) { }
+#endif
+
 /*
  * Map the spin_lock functions to the raw variants for PREEMPT_RT=n
  */
 
 static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
 {
+	check_spinlock_context();
 	return &lock->rlock;
 }
 
diff --git a/kernel/include/linux/spinlock_api_up.h b/kernel/include/linux/spinlock_api_up.h
index d0d188861a..6895779e81 100644
--- a/kernel/include/linux/spinlock_api_up.h
+++ b/kernel/include/linux/spinlock_api_up.h
@@ -30,21 +30,33 @@
 #define __LOCK(lock) \
   do { preempt_disable(); ___LOCK(lock); } while (0)
 
+#define __HARD_LOCK(lock) \
+  do { ___LOCK(lock); } while (0)
+
 #define __LOCK_BH(lock) \
   do { __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_LOCK_OFFSET); ___LOCK(lock); } while (0)
 
 #define __LOCK_IRQ(lock) \
   do { local_irq_disable(); __LOCK(lock); } while (0)
 
+#define __HARD_LOCK_IRQ(lock) \
+  do { hard_local_irq_disable(); __HARD_LOCK(lock); } while (0)
+
 #define __LOCK_IRQSAVE(lock, flags) \
   do { local_irq_save(flags); __LOCK(lock); } while (0)
 
+#define __HARD_LOCK_IRQSAVE(lock, flags) \
+  do { flags = hard_local_irq_save(); __HARD_LOCK(lock); } while (0)
+
 #define ___UNLOCK(lock) \
   do { __release(lock); (void)(lock); } while (0)
 
 #define __UNLOCK(lock) \
   do { preempt_enable(); ___UNLOCK(lock); } while (0)
 
+#define __HARD_UNLOCK(lock) \
+  do { ___UNLOCK(lock); } while (0)
+
 #define __UNLOCK_BH(lock) \
   do { __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_LOCK_OFFSET); \
        ___UNLOCK(lock); } while (0)
@@ -52,9 +64,15 @@
 #define __UNLOCK_IRQ(lock) \
   do { local_irq_enable(); __UNLOCK(lock); } while (0)
 
+#define __HARD_UNLOCK_IRQ(lock) \
+  do { hard_local_irq_enable(); __HARD_UNLOCK(lock); } while (0)
+
 #define __UNLOCK_IRQRESTORE(lock, flags) \
   do { local_irq_restore(flags); __UNLOCK(lock); } while (0)
 
+#define __HARD_UNLOCK_IRQRESTORE(lock, flags) \
+  do { hard_local_irq_restore(flags); __HARD_UNLOCK(lock); } while (0)
+
 #define _raw_spin_lock(lock)			__LOCK(lock)
 #define _raw_spin_lock_nested(lock, subclass)	__LOCK(lock)
 #define _raw_read_lock(lock)			__LOCK(lock)
diff --git a/kernel/include/linux/spinlock_pipeline.h b/kernel/include/linux/spinlock_pipeline.h
new file mode 100644
index 0000000000..1652735f15
--- /dev/null
+++ b/kernel/include/linux/spinlock_pipeline.h
@@ -0,0 +1,387 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#ifndef __LINUX_SPINLOCK_PIPELINE_H
+#define __LINUX_SPINLOCK_PIPELINE_H
+
+#ifndef __LINUX_SPINLOCK_H
+# error "Please don't include this file directly. Use spinlock.h."
+#endif
+
+#include <dovetail/spinlock.h>
+
+#define hard_spin_lock_irqsave(__rlock, __flags)		\
+	do {							\
+		(__flags) = __hard_spin_lock_irqsave(__rlock);	\
+	} while (0)
+
+#define hard_spin_trylock_irqsave(__rlock, __flags)			\
+	({								\
+		int __locked;						\
+		(__flags) = __hard_spin_trylock_irqsave(__rlock, &__locked); \
+		__locked;						\
+	})
+
+#define hybrid_spin_lock_init(__rlock)	hard_spin_lock_init(__rlock)
+
+/*
+ * CAUTION: We don't want the hand-coded irq-enable of
+ * do_raw_spin_lock_flags(), hard locked sections assume that
+ * interrupts are not re-enabled during lock-acquire.
+ */
+#define hard_lock_acquire(__rlock, __try, __ip)				\
+	do {								\
+		hard_spin_lock_prepare(__rlock);			\
+		if (irq_pipeline_debug_locking()) {			\
+			spin_acquire(&(__rlock)->dep_map, 0, __try, __ip); \
+			LOCK_CONTENDED(__rlock, do_raw_spin_trylock, do_raw_spin_lock); \
+		} else {						\
+			do_raw_spin_lock(__rlock);			\
+		}							\
+	} while (0)
+
+#define hard_lock_acquire_nested(__rlock, __subclass, __ip)		\
+	do {								\
+		hard_spin_lock_prepare(__rlock);			\
+		if (irq_pipeline_debug_locking()) {			\
+			spin_acquire(&(__rlock)->dep_map, __subclass, 0, __ip); \
+			LOCK_CONTENDED(__rlock, do_raw_spin_trylock, do_raw_spin_lock); \
+		} else {						\
+			do_raw_spin_lock(__rlock);			\
+		}							\
+	} while (0)
+
+#define hard_trylock_acquire(__rlock, __try, __ip)			\
+	do {								\
+		if (irq_pipeline_debug_locking())			\
+			spin_acquire(&(__rlock)->dep_map, 0, __try, __ip); \
+	} while (0)
+
+#define hard_lock_release(__rlock, __ip)				\
+	do {								\
+		if (irq_pipeline_debug_locking())			\
+			spin_release(&(__rlock)->dep_map, __ip);	\
+		do_raw_spin_unlock(__rlock);				\
+		hard_spin_unlock_finish(__rlock);			\
+	} while (0)
+
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+
+#ifdef CONFIG_DEBUG_SPINLOCK
+/*
+ * Hard spinlocks are not checked for invalid wait context in-band
+ * wise (LD_WAIT_INV). We could be smarter with handling a specific
+ * wait type for them, so that we could detect hard_spin_lock ->
+ * {raw_}spin_lock for instance, but we already have
+ * check_inband_stage() calls all over the place in the latter API, so
+ * that kind of misusage would be detected regardless.
+ */
+#define hard_spin_lock_init(__lock)				\
+	do {							\
+		static struct lock_class_key __key;		\
+		__raw_spin_lock_init((raw_spinlock_t *)__lock, #__lock, &__key, LD_WAIT_INV); \
+	} while (0)
+#else
+#define hard_spin_lock_init(__rlock)				\
+	do { *(__rlock) = __RAW_SPIN_LOCK_UNLOCKED(__rlock); } while (0)
+#endif
+
+/*
+ * XXX: no preempt_enable/disable when hard locking.
+ */
+
+static inline
+void hard_spin_lock(struct raw_spinlock *rlock)
+{
+	hard_lock_acquire(rlock, 0, _THIS_IP_);
+}
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+static inline
+void hard_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
+{
+	hard_lock_acquire_nested(rlock, subclass, _THIS_IP_);
+}
+#else
+static inline
+void hard_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
+{
+	hard_spin_lock(rlock);
+}
+#endif
+
+static inline
+void hard_spin_unlock(struct raw_spinlock *rlock)
+{
+	hard_lock_release(rlock, _THIS_IP_);
+}
+
+static inline
+void hard_spin_lock_irq(struct raw_spinlock *rlock)
+{
+	hard_local_irq_disable();
+	hard_lock_acquire(rlock, 0, _THIS_IP_);
+}
+
+static inline
+void hard_spin_unlock_irq(struct raw_spinlock *rlock)
+{
+	hard_lock_release(rlock, _THIS_IP_);
+	hard_local_irq_enable();
+}
+
+static inline
+void hard_spin_unlock_irqrestore(struct raw_spinlock *rlock,
+				 unsigned long flags)
+{
+	hard_lock_release(rlock, _THIS_IP_);
+	hard_local_irq_restore(flags);
+}
+
+static inline
+unsigned long __hard_spin_lock_irqsave(struct raw_spinlock *rlock)
+{
+	unsigned long flags = hard_local_irq_save();
+
+	hard_lock_acquire(rlock, 0, _THIS_IP_);
+
+	return flags;
+}
+
+static inline
+int hard_spin_trylock(struct raw_spinlock *rlock)
+{
+	hard_spin_trylock_prepare(rlock);
+
+	if (do_raw_spin_trylock(rlock)) {
+		hard_trylock_acquire(rlock, 1, _THIS_IP_);
+		return 1;
+	}
+
+	hard_spin_trylock_fail(rlock);
+
+	return 0;
+}
+
+static inline
+unsigned long __hard_spin_trylock_irqsave(struct raw_spinlock *rlock,
+					  int *locked)
+{
+	unsigned long flags = hard_local_irq_save();
+	*locked = hard_spin_trylock(rlock);
+	return *locked ? flags : ({ hard_local_irq_restore(flags); flags; });
+}
+
+static inline
+int hard_spin_trylock_irq(struct raw_spinlock *rlock)
+{
+	hard_local_irq_disable();
+	return hard_spin_trylock(rlock) ? : ({ hard_local_irq_enable(); 0; });
+}
+
+static inline
+int hard_spin_is_locked(struct raw_spinlock *rlock)
+{
+	return arch_spin_is_locked(&rlock->raw_lock);
+}
+
+static inline
+int hard_spin_is_contended(struct raw_spinlock *rlock)
+{
+#ifdef CONFIG_GENERIC_LOCKBREAK
+	return rlock->break_lock;
+#elif defined(arch_spin_is_contended)
+	return arch_spin_is_contended(&rlock->raw_lock);
+#else
+	return 0;
+#endif
+}
+
+#else  /* !SMP && !DEBUG_SPINLOCK */
+
+#define hard_spin_lock_init(__rlock)	do { (void)(__rlock); } while (0)
+#define hard_spin_lock(__rlock)		__HARD_LOCK(__rlock)
+#define hard_spin_lock_nested(__rlock, __subclass)  \
+	do { __HARD_LOCK(__rlock); (void)(__subclass); } while (0)
+#define hard_spin_unlock(__rlock)	__HARD_UNLOCK(__rlock)
+#define hard_spin_lock_irq(__rlock)	__HARD_LOCK_IRQ(__rlock)
+#define hard_spin_unlock_irq(__rlock)	__HARD_UNLOCK_IRQ(__rlock)
+#define hard_spin_unlock_irqrestore(__rlock, __flags)	\
+	__HARD_UNLOCK_IRQRESTORE(__rlock, __flags)
+#define __hard_spin_lock_irqsave(__rlock)		\
+	({						\
+		unsigned long __flags;			\
+		__HARD_LOCK_IRQSAVE(__rlock, __flags);	\
+		__flags;				\
+	})
+#define __hard_spin_trylock_irqsave(__rlock, __locked)	\
+	({						\
+		unsigned long __flags;			\
+		__HARD_LOCK_IRQSAVE(__rlock, __flags);	\
+		*(__locked) = 1;			\
+		__flags;				\
+	})
+#define hard_spin_trylock(__rlock)	({ __HARD_LOCK(__rlock); 1; })
+#define hard_spin_trylock_irq(__rlock)	({ __HARD_LOCK_IRQ(__rlock); 1; })
+#define hard_spin_is_locked(__rlock)	((void)(__rlock), 0)
+#define hard_spin_is_contended(__rlock)	((void)(__rlock), 0)
+#endif	/* !SMP && !DEBUG_SPINLOCK */
+
+/*
+ * In the pipeline entry context, the regular preemption and root
+ * stall logic do not apply since we may actually have preempted any
+ * critical section of the kernel which is protected by regular
+ * locking (spin or stall), or we may even have preempted the
+ * out-of-band stage. Therefore, we just need to grab the raw spinlock
+ * underlying a hybrid spinlock to exclude other CPUs.
+ *
+ * NOTE: When entering the pipeline, IRQs are already hard disabled.
+ */
+
+void __hybrid_spin_lock(struct raw_spinlock *rlock);
+void __hybrid_spin_lock_nested(struct raw_spinlock *rlock, int subclass);
+
+static inline void hybrid_spin_lock(struct raw_spinlock *rlock)
+{
+	if (in_pipeline())
+		hard_lock_acquire(rlock, 0, _THIS_IP_);
+	else
+		__hybrid_spin_lock(rlock);
+}
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+static inline
+void hybrid_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
+{
+	if (in_pipeline())
+		hard_lock_acquire_nested(rlock, subclass, _THIS_IP_);
+	else
+		__hybrid_spin_lock_nested(rlock, subclass);
+}
+#else
+static inline
+void hybrid_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
+{
+	hybrid_spin_lock(rlock);
+}
+#endif
+
+void __hybrid_spin_unlock(struct raw_spinlock *rlock);
+
+static inline void hybrid_spin_unlock(struct raw_spinlock *rlock)
+{
+	if (in_pipeline())
+		hard_lock_release(rlock, _THIS_IP_);
+	else
+		__hybrid_spin_unlock(rlock);
+}
+
+void __hybrid_spin_lock_irq(struct raw_spinlock *rlock);
+
+static inline void hybrid_spin_lock_irq(struct raw_spinlock *rlock)
+{
+	if (in_pipeline())
+		hard_lock_acquire(rlock, 0, _THIS_IP_);
+	else
+		__hybrid_spin_lock_irq(rlock);
+}
+
+void __hybrid_spin_unlock_irq(struct raw_spinlock *rlock);
+
+static inline void hybrid_spin_unlock_irq(struct raw_spinlock *rlock)
+{
+	if (in_pipeline())
+		hard_lock_release(rlock, _THIS_IP_);
+	else
+		__hybrid_spin_unlock_irq(rlock);
+}
+
+unsigned long __hybrid_spin_lock_irqsave(struct raw_spinlock *rlock);
+
+#define hybrid_spin_lock_irqsave(__rlock, __flags)			\
+	do {								\
+		if (in_pipeline()) {					\
+			hard_lock_acquire(__rlock, 0, _THIS_IP_);	\
+			(__flags) = hard_local_save_flags();		\
+		} else							\
+			(__flags) = __hybrid_spin_lock_irqsave(__rlock); \
+	} while (0)
+
+void __hybrid_spin_unlock_irqrestore(struct raw_spinlock *rlock,
+				      unsigned long flags);
+
+static inline void hybrid_spin_unlock_irqrestore(struct raw_spinlock *rlock,
+						  unsigned long flags)
+{
+
+	if (in_pipeline())
+		hard_lock_release(rlock, _THIS_IP_);
+	else
+		__hybrid_spin_unlock_irqrestore(rlock, flags);
+}
+
+int __hybrid_spin_trylock(struct raw_spinlock *rlock);
+
+static inline int hybrid_spin_trylock(struct raw_spinlock *rlock)
+{
+	if (in_pipeline()) {
+		hard_spin_trylock_prepare(rlock);
+		if (do_raw_spin_trylock(rlock)) {
+			hard_trylock_acquire(rlock, 1, _THIS_IP_);
+			return 1;
+		}
+		hard_spin_trylock_fail(rlock);
+		return 0;
+	}
+
+	return __hybrid_spin_trylock(rlock);
+}
+
+int __hybrid_spin_trylock_irqsave(struct raw_spinlock *rlock,
+				   unsigned long *flags);
+
+#define hybrid_spin_trylock_irqsave(__rlock, __flags)			\
+	({								\
+		int __ret = 1;						\
+		if (in_pipeline()) {					\
+			hard_spin_trylock_prepare(__rlock);		\
+			if (do_raw_spin_trylock(__rlock)) {		\
+				hard_trylock_acquire(__rlock, 1, _THIS_IP_); \
+				(__flags) = hard_local_save_flags();	\
+			} else {					\
+				hard_spin_trylock_fail(__rlock);	\
+				__ret = 0;				\
+			}						\
+		} else {						\
+			__ret = __hybrid_spin_trylock_irqsave(__rlock, &(__flags)); \
+		}							\
+		__ret;							\
+	})
+
+static inline int hybrid_spin_trylock_irq(struct raw_spinlock *rlock)
+{
+	unsigned long flags;
+	return hybrid_spin_trylock_irqsave(rlock, flags);
+}
+
+static inline
+int hybrid_spin_is_locked(struct raw_spinlock *rlock)
+{
+	return hard_spin_is_locked(rlock);
+}
+
+static inline
+int hybrid_spin_is_contended(struct raw_spinlock *rlock)
+{
+	return hard_spin_is_contended(rlock);
+}
+
+#ifdef CONFIG_DEBUG_IRQ_PIPELINE
+void check_spinlock_context(void);
+#else
+static inline void check_spinlock_context(void) { }
+#endif
+
+#endif /* __LINUX_SPINLOCK_PIPELINE_H */
diff --git a/kernel/include/linux/spinlock_types.h b/kernel/include/linux/spinlock_types.h
index b981caafe8..c385825be6 100644
--- a/kernel/include/linux/spinlock_types.h
+++ b/kernel/include/linux/spinlock_types.h
@@ -43,9 +43,15 @@ typedef struct raw_spinlock {
 		.name = #lockname,			\
 		.wait_type_inner = LD_WAIT_CONFIG,	\
 	}
+# define HARD_SPIN_DEP_MAP_INIT(lockname)		\
+	.dep_map = {					\
+		.name = #lockname,			\
+		.wait_type_inner = LD_WAIT_INV,		\
+	}
 #else
 # define RAW_SPIN_DEP_MAP_INIT(lockname)
 # define SPIN_DEP_MAP_INIT(lockname)
+# define HARD_SPIN_DEP_MAP_INIT(lockname)
 #endif
 
 #ifdef CONFIG_DEBUG_SPINLOCK
@@ -96,6 +102,154 @@ typedef struct spinlock {
 
 #define DEFINE_SPINLOCK(x)	spinlock_t x = __SPIN_LOCK_UNLOCKED(x)
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+void __bad_spinlock_type(void);
+
+#define __RAWLOCK(x) ((struct raw_spinlock *)(x))
+
+#define LOCK_ALTERNATIVES(__lock, __base_op, __raw_form, __args...)	\
+	do {								\
+		if (__builtin_types_compatible_p(typeof(__lock),	\
+						 raw_spinlock_t *))	\
+			__raw_form;					\
+		else if (__builtin_types_compatible_p(typeof(__lock),	\
+						 hard_spinlock_t *))	\
+			hard_ ## __base_op(__RAWLOCK(__lock), ##__args); \
+		else if (__builtin_types_compatible_p(typeof(__lock),	\
+						 hybrid_spinlock_t *))	\
+			hybrid_ ## __base_op(__RAWLOCK(__lock), ##__args); \
+		else							\
+			__bad_spinlock_type();				\
+	} while (0)
+
+#define LOCK_ALTERNATIVES_RET(__lock, __base_op, __raw_form, __args...) \
+	({								\
+		long __ret = 0;						\
+		if (__builtin_types_compatible_p(typeof(__lock),	\
+						 raw_spinlock_t *))	\
+			__ret = __raw_form;				\
+		else if (__builtin_types_compatible_p(typeof(__lock),	\
+						 hard_spinlock_t *))	\
+			__ret = hard_ ## __base_op(__RAWLOCK(__lock), ##__args); \
+		else if (__builtin_types_compatible_p(typeof(__lock),	\
+						 hybrid_spinlock_t *))	\
+			__ret = hybrid_ ## __base_op(__RAWLOCK(__lock), ##__args); \
+		else							\
+			__bad_spinlock_type();				\
+		__ret;							\
+	})
+
+#define LOCKDEP_ALT_DEPMAP(__lock)					\
+	({								\
+		struct lockdep_map *__ret;				\
+		if (__builtin_types_compatible_p(typeof(&(__lock)->dep_map), \
+						 struct phony_lockdep_map *)) \
+			__ret = &__RAWLOCK(__lock)->dep_map;		\
+		else							\
+			__ret = (struct lockdep_map *)(&(__lock)->dep_map); \
+		__ret;							\
+	})
+
+#define LOCKDEP_HARD_DEBUG(__lock, __nodebug, __debug)	\
+	do {						\
+		if (__builtin_types_compatible_p(typeof(__lock),	\
+						raw_spinlock_t *) ||	\
+			irq_pipeline_debug_locking()) {			\
+			__debug;			\
+		} else {				\
+			__nodebug;			\
+		}					\
+	} while (0)
+
+#define LOCKDEP_HARD_DEBUG_RET(__lock, __nodebug, __debug)	\
+	({						\
+		typeof(__nodebug) __ret;		\
+		if (__builtin_types_compatible_p(typeof(__lock),	\
+						raw_spinlock_t *) ||	\
+			irq_pipeline_debug_locking()) {			\
+			__ret = (__debug);		\
+		} else {				\
+			__ret = (__nodebug);		\
+		}					\
+		__ret;					\
+	})
+
+#define __HARD_SPIN_LOCK_INITIALIZER(x)	{			\
+		.rlock = {					\
+			.raw_lock = __ARCH_SPIN_LOCK_UNLOCKED,	\
+			SPIN_DEBUG_INIT(x)			\
+			HARD_SPIN_DEP_MAP_INIT(x)		\
+		}						\
+	}
+
+#define __HARD_SPIN_LOCK_UNLOCKED(x)	\
+	(hard_spinlock_t) __HARD_SPIN_LOCK_INITIALIZER(x)
+
+#define DEFINE_HARD_SPINLOCK(x)  	hard_spinlock_t x = __HARD_SPIN_LOCK_UNLOCKED(x)
+
+#define DECLARE_HARD_SPINLOCK(x)	hard_spinlock_t x
+
+/*
+ * The presence of a phony depmap is tested by LOCKDEP_ALT_DEPMAP() to
+ * locate the real depmap without enumerating every spinlock type
+ * which may contain one.
+ */
+struct phony_lockdep_map { };
+
+typedef struct hard_spinlock {
+	/* XXX: offset_of(struct hard_spinlock, rlock) == 0 */
+	struct raw_spinlock rlock;
+	struct phony_lockdep_map dep_map;
+} hard_spinlock_t;
+
+#define DEFINE_MUTABLE_SPINLOCK(x)	hybrid_spinlock_t x = {	\
+		.rlock = __RAW_SPIN_LOCK_UNLOCKED(x),			\
+	}
+
+#define DECLARE_MUTABLE_SPINLOCK(x)	hybrid_spinlock_t x
+
+typedef struct hybrid_spinlock {
+	/* XXX: offset_of(struct hybrid_spinlock, rlock) == 0 */
+	struct raw_spinlock rlock;
+	unsigned long hwflags;
+	struct phony_lockdep_map dep_map;
+} hybrid_spinlock_t;
+
+#else
+
+typedef raw_spinlock_t hard_spinlock_t;
+
+typedef raw_spinlock_t hybrid_spinlock_t;
+
+#define LOCK_ALTERNATIVES(__lock, __base_op, __raw_form, __args...)	\
+	__raw_form
+
+#define LOCK_ALTERNATIVES_RET(__lock, __base_op, __raw_form, __args...) \
+	__raw_form
+
+#define LOCKDEP_ALT_DEPMAP(__lock)	(&(__lock)->dep_map)
+
+#define LOCKDEP_HARD_DEBUG(__lock, __nondebug, __debug)		do { __debug; } while (0)
+
+#define LOCKDEP_HARD_DEBUG_RET(__lock, __nondebug, __debug)	({ __debug; })
+
+#define DEFINE_HARD_SPINLOCK(x)		DEFINE_RAW_SPINLOCK(x)
+
+#define DECLARE_HARD_SPINLOCK(x)	raw_spinlock_t x
+
+#define DEFINE_MUTABLE_SPINLOCK(x)	DEFINE_RAW_SPINLOCK(x)
+
+#define DECLARE_MUTABLE_SPINLOCK(x)	raw_spinlock_t x
+
+#define __RAWLOCK(x) (x)
+
+#define __HARD_SPIN_LOCK_UNLOCKED(__lock)	__RAW_SPIN_LOCK_UNLOCKED(__lock)
+
+#define __HARD_SPIN_LOCK_INITIALIZER(__lock)	__RAW_SPIN_LOCK_UNLOCKED(__lock)
+
+#endif	/* CONFIG_IRQ_PIPELINE */
+
 #include <linux/rwlock_types.h>
 
 #endif /* __LINUX_SPINLOCK_TYPES_H */
diff --git a/kernel/include/linux/stop_machine.h b/kernel/include/linux/stop_machine.h
index ddafb3cf4f..82c56dface 100644
--- a/kernel/include/linux/stop_machine.h
+++ b/kernel/include/linux/stop_machine.h
@@ -6,6 +6,7 @@
 #include <linux/cpumask.h>
 #include <linux/smp.h>
 #include <linux/list.h>
+#include <linux/interrupt.h>
 
 /*
  * stop_cpu[s]() is simplistic per-cpu maximum priority cpu
@@ -143,7 +144,9 @@ static __always_inline int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
 	unsigned long flags;
 	int ret;
 	local_irq_save(flags);
+	hard_irq_disable();
 	ret = fn(data);
+	hard_irq_enable();
 	local_irq_restore(flags);
 	return ret;
 }
diff --git a/kernel/include/linux/thread_info.h b/kernel/include/linux/thread_info.h
index f3040b0b4b..83187b9e50 100644
--- a/kernel/include/linux/thread_info.h
+++ b/kernel/include/linux/thread_info.h
@@ -170,6 +170,72 @@ check_copy_size(const void *addr, size_t bytes, bool is_source)
 static inline void arch_setup_new_exec(void) { }
 #endif
 
+#ifdef ti_local_flags
+/*
+ * If the arch defines a set of per-thread synchronous flags, provide
+ * generic accessors to them.
+ */
+static __always_inline
+void set_ti_local_flags(struct thread_info *ti, unsigned int mask)
+{
+	ti_local_flags(ti) |= mask;
+}
+
+static __always_inline void set_thread_local_flags(unsigned int mask)
+{
+	set_ti_local_flags(current_thread_info(), mask);
+}
+
+static __always_inline
+int test_and_set_ti_local_flags(struct thread_info *ti, unsigned int mask)
+{
+	int old = ti_local_flags(ti) & mask;
+	ti_local_flags(ti) |= mask;
+	return old != 0;
+}
+
+static __always_inline int test_and_set_thread_local_flags(unsigned int mask)
+{
+	return test_and_set_ti_local_flags(current_thread_info(), mask);
+}
+
+static __always_inline
+void clear_ti_local_flags(struct thread_info *ti, unsigned int mask)
+{
+	ti_local_flags(ti) &= ~mask;
+}
+
+static __always_inline
+int test_and_clear_ti_local_flags(struct thread_info *ti, unsigned int mask)
+{
+	int old = ti_local_flags(ti) & mask;
+	ti_local_flags(ti) &= ~mask;
+	return old != 0;
+}
+
+static __always_inline int test_and_clear_thread_local_flags(unsigned int mask)
+{
+	return test_and_clear_ti_local_flags(current_thread_info(), mask);
+}
+
+static __always_inline void clear_thread_local_flags(unsigned int mask)
+{
+	clear_ti_local_flags(current_thread_info(), mask);
+}
+
+static __always_inline
+bool test_ti_local_flags(struct thread_info *ti, unsigned int mask)
+{
+	return (ti_local_flags(ti) & mask) != 0;
+}
+
+static __always_inline bool test_thread_local_flags(unsigned int mask)
+{
+	return test_ti_local_flags(current_thread_info(), mask);
+}
+
+#endif	/* ti_local_flags */
+
 #endif	/* __KERNEL__ */
 
 #endif /* _LINUX_THREAD_INFO_H */
diff --git a/kernel/include/linux/tick.h b/kernel/include/linux/tick.h
index 7340613c7e..b9896c021b 100644
--- a/kernel/include/linux/tick.h
+++ b/kernel/include/linux/tick.h
@@ -20,6 +20,14 @@ extern void tick_suspend_local(void);
 extern void tick_resume_local(void);
 extern void tick_handover_do_timer(void);
 extern void tick_cleanup_dead_cpu(int cpu);
+
+#ifdef CONFIG_IRQ_PIPELINE
+int tick_install_proxy(void (*setup_proxy)(struct clock_proxy_device *dev),
+		const struct cpumask *cpumask);
+void tick_uninstall_proxy(const struct cpumask *cpumask);
+void tick_notify_proxy(void);
+#endif
+
 #else /* CONFIG_GENERIC_CLOCKEVENTS */
 static inline void tick_init(void) { }
 static inline void tick_suspend_local(void) { }
diff --git a/kernel/include/linux/tracepoint.h b/kernel/include/linux/tracepoint.h
index c51a00210f..c8629cd788 100644
--- a/kernel/include/linux/tracepoint.h
+++ b/kernel/include/linux/tracepoint.h
@@ -175,6 +175,10 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
  * The reason for this is to handle the "void" prototype. If a tracepoint
  * has a "void" prototype, then it is invalid to declare a function
  * as "(void *, void)".
+ *
+ * IRQ pipeline: we may not depend on RCU for data which may be
+ * manipulated from the out-of-band stage, so rcuidle has to be false
+ * if running_oob().
  */
 #define __DO_TRACE(name, proto, args, cond, rcuidle)			\
 	do {								\
@@ -223,7 +227,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
 			__DO_TRACE(name,				\
 				TP_PROTO(data_proto),			\
 				TP_ARGS(data_args),			\
-				TP_CONDITION(cond), 1);			\
+				TP_CONDITION(cond), running_inband());	\
 	}
 #else
 #define __DECLARE_TRACE_RCU(name, proto, args, cond, data_proto, data_args)
diff --git a/kernel/include/linux/vmalloc.h b/kernel/include/linux/vmalloc.h
index 167a9533d9..761ed17e6f 100644
--- a/kernel/include/linux/vmalloc.h
+++ b/kernel/include/linux/vmalloc.h
@@ -244,6 +244,7 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
 
 int register_vmap_purge_notifier(struct notifier_block *nb);
 int unregister_vmap_purge_notifier(struct notifier_block *nb);
+void arch_advertise_page_mapping(unsigned long start, unsigned long end);
 
 /* Allow disabling lazy TLB flushing */
 extern bool lazy_vunmap_enable;
diff --git a/kernel/include/linux/wakeup_reason.h b/kernel/include/linux/wakeup_reason.h
index 54f5caaa5c..2fe0fd07a7 100644
--- a/kernel/include/linux/wakeup_reason.h
+++ b/kernel/include/linux/wakeup_reason.h
@@ -20,7 +20,7 @@
 
 #define MAX_SUSPEND_ABORT_LEN 256
 
-#ifdef CONFIG_SUSPEND
+#if IS_ENABLED(CONFIG_SUSPEND) && !IS_ENABLED(CONFIG_DOVETAIL)
 void log_irq_wakeup_reason(int irq);
 void log_threaded_irq_wakeup_reason(int irq, int parent_irq);
 void log_suspend_abort_reason(const char *fmt, ...);
diff --git a/kernel/include/net/netoob.h b/kernel/include/net/netoob.h
new file mode 100644
index 0000000000..907376ad40
--- /dev/null
+++ b/kernel/include/net/netoob.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _NET_OOBNET_H
+#define _NET_OOBNET_H
+
+#include <dovetail/netdevice.h>
+
+/* Device supports direct out-of-band operations (RX & TX) */
+#define IFF_OOB_CAPABLE		BIT(0)
+/* Device is an out-of-band port */
+#define IFF_OOB_PORT		BIT(1)
+
+struct oob_netdev_context {
+	int flags;
+	struct oob_netdev_state dev_state;
+};
+
+#endif /* !_NET_OOBNET_H */
diff --git a/kernel/include/net/sock.h b/kernel/include/net/sock.h
index c604052e4a..ea5f2fa291 100644
--- a/kernel/include/net/sock.h
+++ b/kernel/include/net/sock.h
@@ -540,6 +540,9 @@ struct sock {
 	ANDROID_KABI_RESERVE(8);
 
 	ANDROID_OEM_DATA(1);
+#ifdef CONFIG_NET_OOB
+	void			*oob_data;
+#endif
 };
 
 enum sk_pacing {
diff --git a/kernel/include/trace/events/irq.h b/kernel/include/trace/events/irq.h
index bb70f46656..4642f45f29 100644
--- a/kernel/include/trace/events/irq.h
+++ b/kernel/include/trace/events/irq.h
@@ -100,6 +100,48 @@ TRACE_EVENT(irq_handler_exit,
 		  __entry->irq, __entry->ret ? "handled" : "unhandled")
 );
 
+/**
+ * irq_pipeline_entry - called when an external irq enters the pipeline
+ * @irq: irq number
+ */
+TRACE_EVENT(irq_pipeline_entry,
+
+	TP_PROTO(int irq),
+
+	TP_ARGS(irq),
+
+	TP_STRUCT__entry(
+		__field(	int,	irq		)
+	),
+
+	TP_fast_assign(
+		__entry->irq = irq;
+	),
+
+	TP_printk("irq=%d", __entry->irq)
+);
+
+/**
+ * irq_pipeline_exit - called when an external irq leaves the pipeline
+ * @irq: irq number
+ */
+TRACE_EVENT(irq_pipeline_exit,
+
+	TP_PROTO(int irq),
+
+	TP_ARGS(irq),
+
+	TP_STRUCT__entry(
+		__field(	int,	irq		)
+	),
+
+	TP_fast_assign(
+		__entry->irq = irq;
+	),
+
+	TP_printk("irq=%d", __entry->irq)
+);
+
 DECLARE_EVENT_CLASS(softirq,
 
 	TP_PROTO(unsigned int vec_nr),
diff --git a/kernel/include/uapi/asm-generic/dovetail.h b/kernel/include/uapi/asm-generic/dovetail.h
new file mode 100644
index 0000000000..795aa38443
--- /dev/null
+++ b/kernel/include/uapi/asm-generic/dovetail.h
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef __ASM_GENERIC_DOVETAIL_H
+#define __ASM_GENERIC_DOVETAIL_H
+
+#define __OOB_SYSCALL_BIT	0x10000000
+
+#endif /* !__ASM_GENERIC_DOVETAIL_H */
diff --git a/kernel/include/uapi/asm-generic/fcntl.h b/kernel/include/uapi/asm-generic/fcntl.h
index 9dc0bf0c5a..11415c6bf9 100644
--- a/kernel/include/uapi/asm-generic/fcntl.h
+++ b/kernel/include/uapi/asm-generic/fcntl.h
@@ -89,6 +89,15 @@
 #define __O_TMPFILE	020000000
 #endif
 
+/*
+ * Tells the open call that out-of-band operations should be enabled
+ * for the file (if supported). Can also be passed along to socket(2)
+ * via the type argument as SOCK_OOB.
+ */
+#ifndef O_OOB
+#define O_OOB		010000000000
+#endif
+
 /* a horrid kludge trying to make sure that this will fail on old kernels */
 #define O_TMPFILE (__O_TMPFILE | O_DIRECTORY)
 #define O_TMPFILE_MASK (__O_TMPFILE | O_DIRECTORY | O_CREAT)      
diff --git a/kernel/include/uapi/linux/clocksource.h b/kernel/include/uapi/linux/clocksource.h
new file mode 100644
index 0000000000..a0a1c27473
--- /dev/null
+++ b/kernel/include/uapi/linux/clocksource.h
@@ -0,0 +1,33 @@
+/*
+ * Definitions for user-mappable clock sources.
+ *
+ * Gilles Chanteperdrix <gilles.chanteperdrix@xenomai.org>
+ */
+#ifndef _UAPI_LINUX_CLOCKSOURCE_H
+#define _UAPI_LINUX_CLOCKSOURCE_H
+
+enum clksrc_user_mmio_type {
+	CLKSRC_MMIO_L_UP,
+	CLKSRC_MMIO_L_DOWN,
+	CLKSRC_MMIO_W_UP,
+	CLKSRC_MMIO_W_DOWN,
+	CLKSRC_DMMIO_L_UP,
+	CLKSRC_DMMIO_W_UP,
+
+	CLKSRC_MMIO_TYPE_NR,
+};
+
+struct clksrc_user_mmio_info {
+	enum clksrc_user_mmio_type type;
+	void *reg_lower;
+	unsigned int mask_lower;
+	unsigned int bits_lower;
+	void *reg_upper;
+	unsigned int mask_upper;
+};
+
+#define CLKSRC_USER_MMIO_MAX 16
+
+#define CLKSRC_USER_MMIO_MAP _IOWR(0xC1, 0, struct clksrc_user_mmio_info)
+
+#endif /* _UAPI_LINUX_CLOCKSOURCE_H */
diff --git a/kernel/include/vdso/datapage.h b/kernel/include/vdso/datapage.h
index 73eb622e76..e2cd1e8884 100644
--- a/kernel/include/vdso/datapage.h
+++ b/kernel/include/vdso/datapage.h
@@ -106,9 +106,34 @@ struct vdso_data {
 	u32			hrtimer_res;
 	u32			__unused;
 
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+	u32			cs_type_seq;
+	char			cs_mmdev[16];
+#endif
+ 
 	struct arch_vdso_data	arch_data;
 };
 
+#if defined(CONFIG_GENERIC_CLOCKSOURCE_VDSO) && !defined(ENABLE_COMPAT_VDSO)
+
+#include <linux/clocksource.h>
+
+struct clksrc_info;
+
+typedef u64 vdso_read_cycles_t(const struct clksrc_info *info);
+
+struct clksrc_info {
+	vdso_read_cycles_t *read_cycles;
+	struct clksrc_user_mmio_info mmio;
+};
+
+struct vdso_priv {
+	u32 current_cs_type_seq;
+	struct clksrc_info clksrc_info[CLOCKSOURCE_VDSO_MMIO + CLKSRC_USER_MMIO_MAX];
+};
+
+#endif	/* !CONFIG_GENERIC_CLOCKSOURCE_VDSO */
+
 /*
  * We use the hidden visibility to prevent the compiler from generating a GOT
  * relocation. Not only is going through a GOT useless (the entry couldn't and
diff --git a/kernel/init/Kconfig b/kernel/init/Kconfig
index 2c92f94281..6a86ef1f99 100644
--- a/kernel/init/Kconfig
+++ b/kernel/init/Kconfig
@@ -1531,7 +1531,19 @@ config PRINTK
 config PRINTK_NMI
 	def_bool y
 	depends on PRINTK
-	depends on HAVE_NMI
+	depends on HAVE_NMI || IRQ_PIPELINE
+
+config RAW_PRINTK
+       bool "Enable support for raw printk"
+       default n
+       help
+         This option enables a printk variant called raw_printk() for
+         writing all output unmodified to a raw console channel
+         immediately, without any header or preparation whatsoever,
+         usable from any context.
+
+	 Unlike early_printk() console devices, raw_printk() devices
+         can live past the boot sequence.
 
 config BUG
 	bool "BUG() support" if EXPERT
diff --git a/kernel/init/Makefile b/kernel/init/Makefile
index 6bc37f64b3..ce21edc2d6 100644
--- a/kernel/init/Makefile
+++ b/kernel/init/Makefile
@@ -34,4 +34,4 @@ include/generated/compile.h: FORCE
 	@$($(quiet)chk_compile.h)
 	$(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@	\
 	"$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)"	\
-	"$(CONFIG_PREEMPT_RT)" $(CONFIG_CC_VERSION_TEXT) "$(LD)"
+	"$(CONFIG_PREEMPT_RT)" "$(CONFIG_IRQ_PIPELINE)" $(CONFIG_CC_VERSION_TEXT) "$(LD)"
diff --git a/kernel/init/main.c b/kernel/init/main.c
index 45ca352189..6443f34b7a 100644
--- a/kernel/init/main.c
+++ b/kernel/init/main.c
@@ -52,6 +52,7 @@
 #include <linux/tick.h>
 #include <linux/sched/isolation.h>
 #include <linux/interrupt.h>
+#include <linux/irq_pipeline.h>
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>
 #include <linux/unistd.h>
@@ -847,13 +848,14 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
 	char *command_line;
 	char *after_dashes;
 
+	stall_inband_nocheck();
 	set_task_stack_end_magic(&init_task);
 	smp_setup_processor_id();
 	debug_objects_early_init();
 
 	cgroup_init_early();
 
-	local_irq_disable();
+	local_irq_disable_full();
 	early_boot_irqs_disabled = true;
 
 	/*
@@ -913,6 +915,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
 	setup_log_buf(0);
 	vfs_caches_init_early();
 	sort_main_extable();
+	irq_pipeline_init_early();
 	trap_init();
 	mm_init();
 
@@ -958,6 +961,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
 	/* init some links before init_ISA_irqs() */
 	early_irq_init();
 	init_IRQ();
+	irq_pipeline_init();
 	tick_init();
 	rcu_init_nohz();
 	init_timers();
@@ -983,7 +987,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
 	WARN(!irqs_disabled(), "Interrupts were enabled early\n");
 
 	early_boot_irqs_disabled = false;
-	local_irq_enable();
+	local_irq_enable_full();
 
 	kmem_cache_init_late();
 
diff --git a/kernel/kernel/Kconfig.dovetail b/kernel/kernel/Kconfig.dovetail
new file mode 100644
index 0000000000..c9ec30d1b4
--- /dev/null
+++ b/kernel/kernel/Kconfig.dovetail
@@ -0,0 +1,23 @@
+
+# DOVETAIL dual-kernel interface
+config HAVE_DOVETAIL
+	bool
+
+# Selecting ARCH_WANT_IRQS_OFF_ACTIVATE_MM in this generic Kconfig
+# portion is ugly, but the whole ARCH_WANT_IRQS_OFF_ACTIVATE_MM logic
+# is a temporary kludge which is meant to disappear anyway. See
+# the related comments in exec_mmap() for details.
+config DOVETAIL
+	bool "Dovetail interface"
+	depends on HAVE_DOVETAIL
+	select IRQ_PIPELINE
+	select ARCH_WANT_IRQS_OFF_ACTIVATE_MM
+	default n
+	help
+	  Activate this option if you want to enable the interface for
+	  running a secondary kernel side-by-side with Linux (aka
+	  "dual kernel" configuration).
+
+config DOVETAIL_LEGACY_SYSCALL_RANGE
+       depends on DOVETAIL
+       def_bool y
diff --git a/kernel/kernel/Makefile b/kernel/kernel/Makefile
index 6ee614db17..6d08ced5cc 100644
--- a/kernel/kernel/Makefile
+++ b/kernel/kernel/Makefile
@@ -106,6 +106,7 @@ obj-$(CONFIG_TRACE_CLOCK) += trace/
 obj-$(CONFIG_RING_BUFFER) += trace/
 obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
+obj-$(CONFIG_DOVETAIL) += dovetail.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
 obj-$(CONFIG_KCSAN) += kcsan/
diff --git a/kernel/kernel/debug/debug_core.c b/kernel/kernel/debug/debug_core.c
index 0f31b22abe..6e034f0087 100644
--- a/kernel/kernel/debug/debug_core.c
+++ b/kernel/kernel/debug/debug_core.c
@@ -111,8 +111,8 @@ static struct kgdb_bkpt		kgdb_break[KGDB_MAX_BREAKPOINTS] = {
  */
 atomic_t			kgdb_active = ATOMIC_INIT(-1);
 EXPORT_SYMBOL_GPL(kgdb_active);
-static DEFINE_RAW_SPINLOCK(dbg_master_lock);
-static DEFINE_RAW_SPINLOCK(dbg_slave_lock);
+static DEFINE_HARD_SPINLOCK(dbg_master_lock);
+static DEFINE_HARD_SPINLOCK(dbg_slave_lock);
 
 /*
  * We use NR_CPUs not PERCPU, in case kgdb is used to debug early
@@ -612,7 +612,7 @@ static int kgdb_cpu_enter(struct kgdb_state *ks, struct pt_regs *regs,
 	 * Interrupts will be restored by the 'trap return' code, except when
 	 * single stepping.
 	 */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	cpu = ks->cpu;
 	kgdb_info[cpu].debuggerinfo = regs;
@@ -666,7 +666,7 @@ static int kgdb_cpu_enter(struct kgdb_state *ks, struct pt_regs *regs,
 			smp_mb__before_atomic();
 			atomic_dec(&slaves_in_kgdb);
 			dbg_touch_watchdogs();
-			local_irq_restore(flags);
+			hard_local_irq_restore(flags);
 			rcu_read_unlock();
 			return 0;
 		}
@@ -685,7 +685,7 @@ static int kgdb_cpu_enter(struct kgdb_state *ks, struct pt_regs *regs,
 		atomic_set(&kgdb_active, -1);
 		raw_spin_unlock(&dbg_master_lock);
 		dbg_touch_watchdogs();
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 		rcu_read_unlock();
 
 		goto acquirelock;
@@ -721,8 +721,11 @@ static int kgdb_cpu_enter(struct kgdb_state *ks, struct pt_regs *regs,
 		atomic_set(ks->send_ready, 1);
 
 	/* Signal the other CPUs to enter kgdb_wait() */
-	else if ((!kgdb_single_step) && kgdb_do_roundup)
+	else if ((!kgdb_single_step) && kgdb_do_roundup && running_inband()) {
+		hard_cond_local_irq_enable();
 		kgdb_roundup_cpus();
+		hard_cond_local_irq_disable();
+	}
 #endif
 
 	/*
@@ -834,7 +837,7 @@ static int kgdb_cpu_enter(struct kgdb_state *ks, struct pt_regs *regs,
 	atomic_set(&kgdb_active, -1);
 	raw_spin_unlock(&dbg_master_lock);
 	dbg_touch_watchdogs();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	rcu_read_unlock();
 
 	return kgdb_info[cpu].ret_state;
@@ -957,7 +960,7 @@ static void kgdb_console_write(struct console *co, const char *s,
 	if (!kgdb_connected || atomic_read(&kgdb_active) != -1 || dbg_kdb_mode)
 		return;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	gdbstub_msg_write(s, count);
 	local_irq_restore(flags);
 }
diff --git a/kernel/kernel/dovetail.c b/kernel/kernel/dovetail.c
new file mode 100644
index 0000000000..4d1e4c35b9
--- /dev/null
+++ b/kernel/kernel/dovetail.c
@@ -0,0 +1,450 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#include <linux/timekeeper_internal.h>
+#include <linux/sched/signal.h>
+#include <linux/irq_pipeline.h>
+#include <linux/dovetail.h>
+#include <asm/unistd.h>
+#include <asm/syscall.h>
+#include <uapi/asm-generic/dovetail.h>
+
+static bool dovetail_enabled;
+
+void __weak arch_inband_task_init(struct task_struct *p)
+{
+}
+
+void inband_task_init(struct task_struct *p)
+{
+	struct thread_info *ti = task_thread_info(p);
+
+	clear_ti_local_flags(ti, _TLF_DOVETAIL|_TLF_OOB|_TLF_OFFSTAGE);
+	arch_inband_task_init(p);
+}
+
+void dovetail_init_altsched(struct dovetail_altsched_context *p)
+{
+	struct task_struct *tsk = current;
+	struct mm_struct *mm = tsk->mm;
+
+	check_inband_stage();
+	p->task = tsk;
+	p->active_mm = mm;
+	p->borrowed_mm = false;
+
+	/*
+	 * Make sure the current process will not share any private
+	 * page with its child upon fork(), sparing it the random
+	 * latency induced by COW. MMF_DOVETAILED is never cleared once
+	 * set. We serialize with dup_mmap() which holds the mm write
+	 * lock.
+	 */
+	if (!(tsk->flags & PF_KTHREAD) &&
+		!test_bit(MMF_DOVETAILED, &mm->flags)) {
+		mmap_write_lock(mm);
+		__set_bit(MMF_DOVETAILED, &mm->flags);
+		mmap_write_unlock(mm);
+	}
+}
+EXPORT_SYMBOL_GPL(dovetail_init_altsched);
+
+void dovetail_start_altsched(void)
+{
+	check_inband_stage();
+	set_thread_local_flags(_TLF_DOVETAIL);
+}
+EXPORT_SYMBOL_GPL(dovetail_start_altsched);
+
+void dovetail_stop_altsched(void)
+{
+	clear_thread_local_flags(_TLF_DOVETAIL);
+	clear_thread_flag(TIF_MAYDAY);
+}
+EXPORT_SYMBOL_GPL(dovetail_stop_altsched);
+
+int __weak handle_oob_syscall(struct pt_regs *regs)
+{
+	return 0;
+}
+
+int __weak handle_pipelined_syscall(struct irq_stage *stage,
+				    struct pt_regs *regs)
+{
+	return 0;	/* i.e. propagate to in-band handler. */
+}
+
+void __weak handle_oob_mayday(struct pt_regs *regs)
+{
+}
+
+static inline
+void call_mayday(struct thread_info *ti, struct pt_regs *regs)
+{
+	clear_ti_thread_flag(ti, TIF_MAYDAY);
+	handle_oob_mayday(regs);
+}
+
+void dovetail_call_mayday(struct pt_regs *regs)
+{
+	struct thread_info *ti = current_thread_info();
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	call_mayday(ti, regs);
+	hard_local_irq_restore(flags);
+}
+
+void inband_retuser_notify(void)
+{
+	clear_thread_flag(TIF_RETUSER);
+	inband_event_notify(INBAND_TASK_RETUSER, current);
+	/* CAUTION: we might have switched out-of-band here. */
+}
+
+int __pipeline_syscall(struct pt_regs *regs)
+{
+	struct thread_info *ti = current_thread_info();
+	struct irq_stage *caller_stage, *target_stage;
+	struct irq_stage_data *p, *this_context;
+	unsigned long flags;
+	int ret = 0;
+
+	/*
+	 * We should definitely not pipeline a syscall through the
+	 * slow path with IRQs off.
+	 */
+	WARN_ON_ONCE(dovetail_debug() && hard_irqs_disabled());
+
+	if (!dovetail_enabled)
+		return 0;
+
+	flags = hard_local_irq_save();
+	caller_stage = current_irq_stage;
+	this_context = current_irq_staged;
+	target_stage = &oob_stage;
+next:
+	p = this_staged(target_stage);
+	set_current_irq_staged(p);
+	hard_local_irq_restore(flags);
+	ret = handle_pipelined_syscall(caller_stage, regs);
+	flags = hard_local_irq_save();
+	/*
+	 * Be careful about stage switching _and_ CPU migration that
+	 * might have happened as a result of handing over the syscall
+	 * to the out-of-band handler.
+	 *
+	 * - if a stage migration is detected, fetch the new
+	 * per-stage, per-CPU context pointer.
+	 *
+	 * - if no stage migration happened, switch back to the
+	 * initial call stage, on a possibly different CPU though.
+	 */
+	if (current_irq_stage != target_stage) {
+		this_context = current_irq_staged;
+	} else {
+		p = this_staged(this_context->stage);
+		set_current_irq_staged(p);
+	}
+
+	if (this_context->stage == &inband_stage) {
+		if (target_stage != &inband_stage && ret == 0) {
+			target_stage = &inband_stage;
+			goto next;
+		}
+		p = this_inband_staged();
+		if (stage_irqs_pending(p))
+			sync_current_irq_stage();
+	} else {
+		if (test_ti_thread_flag(ti, TIF_MAYDAY))
+			call_mayday(ti, regs);
+	}
+
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+static inline bool maybe_oob_syscall(unsigned int nr, struct pt_regs *regs)
+{
+	/*
+	 * Check whether the companion core might be interested in the
+	 * syscall call. If the old syscall form is handled, pass the
+	 * request to the core if __OOB_SYSCALL_BIT is set in
+	 * @nr. Otherwise, only check whether an oob syscall is folded
+	 * into a prctl() request.
+	 */
+	if (IS_ENABLED(CONFIG_DOVETAIL_LEGACY_SYSCALL_RANGE)) {
+		if (nr & __OOB_SYSCALL_BIT)
+			return true;
+	}
+
+	return arch_dovetail_is_syscall(nr) && syscall_get_arg0(regs) & __OOB_SYSCALL_BIT;
+}
+
+int pipeline_syscall(unsigned int nr, struct pt_regs *regs)
+{
+	struct thread_info *ti = current_thread_info();
+	unsigned long local_flags = READ_ONCE(ti_local_flags(ti));
+	int ret;
+
+	WARN_ON_ONCE(dovetail_debug() && hard_irqs_disabled());
+
+	/*
+	 * If the syscall signature belongs to the out-of-band syscall
+	 * set and we are running out-of-band, pass the request
+	 * directly to the companion core by calling the oob syscall
+	 * handler.
+	 *
+	 * Otherwise, if this is an out-of-band syscall or alternate
+	 * scheduling is enabled for the caller, propagate the syscall
+	 * through the pipeline stages, so that:
+	 *
+	 * - the core can manipulate the current execution stage for
+	 * handling the request, which includes switching the current
+	 * thread back to the in-band context if the syscall is a
+	 * native one, or promoting it to the oob stage if handling an
+	 * oob syscall requires this.
+	 *
+	 * - the core can receive the initial oob syscall a thread
+	 * might have to emit for enabling dovetailing from the
+	 * in-band stage.
+	 *
+	 * Native syscalls from common (non-dovetailed) threads are
+	 * not subject to pipelining, but flow down to the in-band
+	 * system call handler directly.
+	 *
+	 * Sanity check: we bark on returning from a syscall on a
+	 * stalled in-band stage, which combined with running with
+	 * hard irqs on might cause interrupts to linger in the log
+	 * after exiting to user.
+	 */
+
+	if ((local_flags & _TLF_OOB) && maybe_oob_syscall(nr, regs)) {
+		ret = handle_oob_syscall(regs);
+		if (!IS_ENABLED(CONFIG_DOVETAIL_LEGACY_SYSCALL_RANGE))
+			WARN_ON_ONCE(dovetail_debug() && !ret);
+		local_flags = READ_ONCE(ti_local_flags(ti));
+		if (likely(ret)) {
+			if (local_flags & _TLF_OOB) {
+				if (test_ti_thread_flag(ti, TIF_MAYDAY))
+					dovetail_call_mayday(regs);
+				return 1; /* don't pass down, no tail work. */
+			} else {
+				WARN_ON_ONCE(dovetail_debug() && irqs_disabled());
+				return -1; /* don't pass down, do tail work. */
+			}
+		}
+	}
+
+	if ((local_flags & _TLF_DOVETAIL) || maybe_oob_syscall(nr, regs)) {
+		ret = __pipeline_syscall(regs);
+		local_flags = READ_ONCE(ti_local_flags(ti));
+		if (local_flags & _TLF_OOB)
+			return 1; /* don't pass down, no tail work. */
+		if (ret) {
+			WARN_ON_ONCE(dovetail_debug() && irqs_disabled());
+			return -1; /* don't pass down, do tail work. */
+		}
+	}
+
+	return 0; /* pass syscall down to the in-band dispatcher. */
+}
+
+void __weak handle_oob_trap_entry(unsigned int trapnr, struct pt_regs *regs)
+{
+}
+
+noinstr void __oob_trap_notify(unsigned int exception,
+			       struct pt_regs *regs)
+{
+	unsigned long flags;
+
+	/*
+	 * We send a notification about exceptions raised over a
+	 * registered oob stage only. The trap_entry handler expects
+	 * hard irqs off on entry. It may demote the current context
+	 * to the in-band stage, may return with hard irqs on.
+	 */
+	if (dovetail_enabled) {
+		set_thread_local_flags(_TLF_OOBTRAP);
+		flags = hard_local_irq_save();
+		instrumentation_begin();
+		handle_oob_trap_entry(exception, regs);
+		instrumentation_end();
+		hard_local_irq_restore(flags);
+	}
+}
+
+void __weak handle_oob_trap_exit(unsigned int trapnr, struct pt_regs *regs)
+{
+}
+
+noinstr void __oob_trap_unwind(unsigned int exception, struct pt_regs *regs)
+{
+	/*
+	 * The trap_exit handler runs only if trap_entry was called
+	 * for the same trap occurrence. It expects hard irqs off on
+	 * entry, may switch the current context back to the oob
+	 * stage. Must return with hard irqs off.
+	 */
+	hard_local_irq_disable();
+	clear_thread_local_flags(_TLF_OOBTRAP);
+	instrumentation_begin();
+	handle_oob_trap_exit(exception, regs);
+	instrumentation_end();
+}
+
+void __weak handle_inband_event(enum inband_event_type event, void *data)
+{
+}
+
+void inband_event_notify(enum inband_event_type event, void *data)
+{
+	check_inband_stage();
+
+	if (dovetail_enabled)
+		handle_inband_event(event, data);
+}
+
+void __weak resume_oob_task(struct task_struct *p)
+{
+}
+
+static void finalize_oob_transition(void) /* hard IRQs off */
+{
+	struct irq_pipeline_data *pd;
+	struct irq_stage_data *p;
+	struct thread_info *ti;
+	struct task_struct *t;
+
+	pd = raw_cpu_ptr(&irq_pipeline);
+	t = pd->task_inflight;
+	if (t == NULL)
+		return;
+
+	/*
+	 * @t which is in flight to the oob stage might have received
+	 * a signal while waiting in off-stage state to be actually
+	 * scheduled out. We can't act upon that signal safely from
+	 * here, we simply let the task complete the migration process
+	 * to the oob stage. The pending signal will be handled when
+	 * the task eventually exits the out-of-band context by the
+	 * converse migration.
+	 */
+	pd->task_inflight = NULL;
+	ti = task_thread_info(t);
+
+	/*
+	 * The transition handler in the companion core assumes the
+	 * oob stage is stalled, fix this up.
+	 */
+	stall_oob();
+	resume_oob_task(t);
+	unstall_oob();
+	p = this_oob_staged();
+	if (stage_irqs_pending(p))
+		/* Current stage (in-band) != p->stage (oob). */
+		sync_irq_stage(p->stage);
+}
+
+void oob_trampoline(void)
+{
+	unsigned long flags;
+
+	check_inband_stage();
+	flags = hard_local_irq_save();
+	finalize_oob_transition();
+	hard_local_irq_restore(flags);
+}
+
+bool inband_switch_tail(void)
+{
+	bool oob;
+
+	check_hard_irqs_disabled();
+
+	/*
+	 * We may run this code either over the inband or oob
+	 * contexts. If inband, we may have a thread blocked in
+	 * dovetail_leave_inband(), waiting for the companion core to
+	 * schedule it back in over the oob context, in which case
+	 * finalize_oob_transition() should take care of it. If oob,
+	 * the core just switched us back, and we may update the
+	 * context markers before returning to context_switch().
+	 *
+	 * Since the preemption count does not reflect the active
+	 * stage yet upon inband -> oob transition, we figure out
+	 * which one we are on by testing _TLF_OFFSTAGE. Having this
+	 * bit set when running the inband switch tail code means that
+	 * we are completing such transition for the current task,
+	 * switched in by dovetail_context_switch() over the oob
+	 * stage. If so, update the context markers appropriately.
+	 */
+	oob = test_thread_local_flags(_TLF_OFFSTAGE);
+	if (oob) {
+		/*
+		 * The companion core assumes a stalled stage on exit
+		 * from dovetail_leave_inband().
+		 */
+		stall_oob();
+		set_thread_local_flags(_TLF_OOB);
+		if (!IS_ENABLED(CONFIG_HAVE_PERCPU_PREEMPT_COUNT)) {
+			WARN_ON_ONCE(dovetail_debug() &&
+				(preempt_count() & STAGE_MASK));
+			preempt_count_add(STAGE_OFFSET);
+		}
+	} else {
+		finalize_oob_transition();
+		hard_local_irq_enable();
+	}
+
+	return oob;
+}
+
+void __weak inband_clock_was_set(void)
+{
+}
+
+void __weak install_inband_fd(unsigned int fd, struct file *file,
+			      struct files_struct *files)
+{
+}
+
+void __weak uninstall_inband_fd(unsigned int fd, struct file *file,
+				struct files_struct *files)
+{
+}
+
+void __weak replace_inband_fd(unsigned int fd, struct file *file,
+			      struct files_struct *files)
+{
+}
+
+int dovetail_start(void)
+{
+	check_inband_stage();
+
+	if (dovetail_enabled)
+		return -EBUSY;
+
+	if (!oob_stage_present())
+		return -EAGAIN;
+
+	dovetail_enabled = true;
+	smp_wmb();
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(dovetail_start);
+
+void dovetail_stop(void)
+{
+	check_inband_stage();
+
+	dovetail_enabled = false;
+	smp_wmb();
+}
+EXPORT_SYMBOL_GPL(dovetail_stop);
diff --git a/kernel/kernel/entry/common.c b/kernel/kernel/entry/common.c
index 09f58853f6..5f043bbe6b 100644
--- a/kernel/kernel/entry/common.c
+++ b/kernel/kernel/entry/common.c
@@ -2,6 +2,7 @@
 
 #include <linux/context_tracking.h>
 #include <linux/entry-common.h>
+#include <linux/irq_pipeline.h>
 #include <linux/livepatch.h>
 #include <linux/audit.h>
 
@@ -71,10 +72,45 @@ static long syscall_trace_enter(struct pt_regs *regs, long syscall,
 	return ret ? : syscall;
 }
 
+static __always_inline void
+syscall_enter_from_user_enable_irqs(void)
+{
+	if (running_inband()) {
+		/*
+		 * If pipelining interrupts, prepare for emulating a
+		 * stall -> unstall transition (we are currently
+		 * unstalled), fixing up the IRQ trace state in order
+		 * to keep lockdep happy (and silent).
+		 */
+		stall_inband_nocheck();
+		hard_cond_local_irq_enable();
+		local_irq_enable();
+	} else {
+		/*
+		 * We are running on the out-of-band stage, don't mess
+		 * with the in-band interrupt state. This is none of
+		 * our business. We may manipulate the hardware state
+		 * only.
+		 */
+		hard_local_irq_enable();
+	}
+}
+
 static __always_inline long
 __syscall_enter_from_user_work(struct pt_regs *regs, long syscall)
 {
 	unsigned long ti_work;
+	int ret;
+
+	/*
+	 * Pipeline the syscall to the companion core if the current
+	 * task wants this. Compiled out if not dovetailing.
+	 */
+	ret = pipeline_syscall(syscall, regs);
+	if (ret > 0)	/* out-of-band, bail out. */
+		return EXIT_SYSCALL_OOB;
+	if (ret < 0)		/* in-band, tail work only. */
+		return EXIT_SYSCALL_TAIL;
 
 	ti_work = READ_ONCE(current_thread_info()->flags);
 	if (ti_work & SYSCALL_ENTER_WORK)
@@ -95,7 +131,7 @@ noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
 	enter_from_user_mode(regs);
 
 	instrumentation_begin();
-	local_irq_enable();
+	syscall_enter_from_user_enable_irqs();
 	ret = __syscall_enter_from_user_work(regs, syscall);
 	instrumentation_end();
 
@@ -106,7 +142,7 @@ noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)
 {
 	enter_from_user_mode(regs);
 	instrumentation_begin();
-	local_irq_enable();
+	syscall_enter_from_user_enable_irqs();
 	instrumentation_end();
 }
 
@@ -121,6 +157,7 @@ noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)
  * 3) Invoke architecture specific last minute exit code, e.g. speculation
  *    mitigations, etc.
  * 4) Tell lockdep that interrupts are enabled
+ * 5) Unstall the in-band stage of the interrupt pipeline if current
  */
 static __always_inline void exit_to_user_mode(void)
 {
@@ -132,6 +169,8 @@ static __always_inline void exit_to_user_mode(void)
 	user_enter_irqoff();
 	arch_exit_to_user_mode();
 	lockdep_hardirqs_on(CALLER_ADDR0);
+	if (running_inband())
+		unstall_inband();
 }
 
 /* Workaround to allow gradual conversion of architecture code */
@@ -156,6 +195,12 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
 
 		local_irq_enable_exit_to_user(ti_work);
 
+		/*
+		 * Check that local_irq_enable_exit_to_user() does the
+		 * right thing when pipelining.
+		 */
+		WARN_ON_ONCE(irq_pipeline_debug() && hard_irqs_disabled());
+
 		if (ti_work & _TIF_NEED_RESCHED)
 			schedule();
 
@@ -182,6 +227,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
 		 * enabled above.
 		 */
 		local_irq_disable_exit_to_user();
+		WARN_ON_ONCE(irq_pipeline_debug() && !hard_irqs_disabled());
 		ti_work = READ_ONCE(current_thread_info()->flags);
 	}
 
@@ -189,17 +235,37 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
 	return ti_work;
 }
 
+static inline bool do_retuser(unsigned long ti_work)
+{
+	if (dovetailing() && (ti_work & _TIF_RETUSER)) {
+		hard_local_irq_enable();
+		inband_retuser_notify();
+		hard_local_irq_disable();
+		/* RETUSER might have switched oob */
+		return running_inband();
+	}
+
+	return false;
+}
+
 static void exit_to_user_mode_prepare(struct pt_regs *regs)
 {
-	unsigned long ti_work = READ_ONCE(current_thread_info()->flags);
+	unsigned long ti_work;
+
+	check_hard_irqs_disabled();
 
 	lockdep_assert_irqs_disabled();
+again:
+	ti_work = READ_ONCE(current_thread_info()->flags);
 
 	if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
 		ti_work = exit_to_user_mode_loop(regs, ti_work);
 
 	arch_exit_to_user_mode_prepare(regs, ti_work);
 
+	if (do_retuser(ti_work))
+		goto again;
+
 	/* Ensure that the address limit is intact and no locks are held */
 	addr_limit_user_check();
 	lockdep_assert_irqs_disabled();
@@ -252,7 +318,7 @@ static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
 
 	if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
 		if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
-			local_irq_enable();
+			local_irq_enable_full();
 	}
 
 	rseq_syscall(regs);
@@ -261,8 +327,15 @@ static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
 	 * Do one-time syscall specific work. If these work items are
 	 * enabled, we want to run them exactly once per syscall exit with
 	 * interrupts enabled.
+	 *
+	 * Dovetail: if this does not look like an in-band syscall, it
+	 * has to belong to the companion core. Typically,
+	 * __OOB_SYSCALL_BIT would be set in this value. Skip the
+	 * work for those syscalls.
 	 */
-	if (unlikely(cached_flags & SYSCALL_EXIT_WORK))
+	if (unlikely((cached_flags & SYSCALL_EXIT_WORK) &&
+		(!irqs_pipelined() ||
+			syscall_get_nr(current, regs) < NR_syscalls)))
 		syscall_exit_work(regs, cached_flags);
 }
 
@@ -278,6 +351,8 @@ __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)
 
 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
 {
+	WARN_ON_ONCE(irq_pipeline_debug() && irqs_disabled());
+	stall_inband_nocheck();
 	enter_from_user_mode(regs);
 }
 
@@ -293,13 +368,37 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
 {
 	irqentry_state_t ret = {
 		.exit_rcu = false,
+#ifdef CONFIG_IRQ_PIPELINE
+		.stage_info = IRQENTRY_INBAND_STALLED,
+#endif
 	};
 
+#ifdef CONFIG_IRQ_PIPELINE
+	if (running_oob()) {
+		WARN_ON_ONCE(irq_pipeline_debug() && oob_irqs_disabled());
+		ret.stage_info = IRQENTRY_OOB;
+		return ret;
+	}
+#endif
+
 	if (user_mode(regs)) {
+#ifdef CONFIG_IRQ_PIPELINE
+		ret.stage_info = IRQENTRY_INBAND_UNSTALLED;
+#endif
 		irqentry_enter_from_user_mode(regs);
 		return ret;
 	}
 
+#ifdef CONFIG_IRQ_PIPELINE
+	/*
+	 * IRQ pipeline: If we trapped from kernel space, the virtual
+	 * state may or may not match the hardware state. Since hard
+	 * irqs are off on entry, we have to stall the in-band stage.
+	 */
+	if (!test_and_stall_inband_nocheck())
+		ret.stage_info = IRQENTRY_INBAND_UNSTALLED;
+#endif
+
 	/*
 	 * If this entry hit the idle task invoke rcu_irq_enter() whether
 	 * RCU is watching or not.
@@ -366,14 +465,91 @@ void irqentry_exit_cond_resched(void)
 	}
 }
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+static inline
+bool irqexit_may_preempt_schedule(irqentry_state_t state,
+				struct pt_regs *regs)
+{
+	return state.stage_info == IRQENTRY_INBAND_UNSTALLED;
+}
+
+#else
+
+static inline
+bool irqexit_may_preempt_schedule(irqentry_state_t state,
+				struct pt_regs *regs)
+{
+	return !regs_irqs_disabled(regs);
+}
+
+#endif
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+static bool irqentry_syncstage(irqentry_state_t state) /* hard irqs off */
+{
+	/*
+	 * If pipelining interrupts, enable in-band IRQs then
+	 * synchronize the interrupt log on exit if:
+	 *
+	 * - irqentry_enter() stalled the stage in order to mirror the
+	 * hardware state.
+	 *
+	 * - we where coming from oob, thus went through a stage migration
+	 * that was caused by taking a CPU exception, e.g., a fault.
+	 *
+	 * We run before preempt_schedule_irq() may be called later on
+	 * by preemptible kernels, so that any rescheduling request
+	 * triggered by in-band IRQ handlers is considered.
+	 */
+	if (state.stage_info == IRQENTRY_INBAND_UNSTALLED ||
+		state.stage_info == IRQENTRY_OOB) {
+		unstall_inband_nocheck();
+		synchronize_pipeline_on_irq();
+		stall_inband_nocheck();
+		return true;
+	}
+
+	return false;
+}
+
+static void irqentry_unstall(void)
+{
+	unstall_inband_nocheck();
+}
+
+#else
+
+static bool irqentry_syncstage(irqentry_state_t state)
+{
+	return false;
+}
+
+static void irqentry_unstall(void)
+{
+}
+
+#endif
+
 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
 {
+	bool synchronized = false;
+
+	if (running_oob())
+		return;
+
 	lockdep_assert_irqs_disabled();
 
 	/* Check whether this returns to user mode */
 	if (user_mode(regs)) {
 		irqentry_exit_to_user_mode(regs);
-	} else if (!regs_irqs_disabled(regs)) {
+		return;
+	}
+
+	synchronized = irqentry_syncstage(state);
+
+	if (irqexit_may_preempt_schedule(state, regs)) {
 		/*
 		 * If RCU was not watching on entry this needs to be done
 		 * carefully and needs the same ordering of lockdep/tracing
@@ -387,7 +563,7 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
 			instrumentation_end();
 			rcu_irq_exit();
 			lockdep_hardirqs_on(CALLER_ADDR0);
-			return;
+			goto out;
 		}
 
 		instrumentation_begin();
@@ -404,6 +580,12 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
 		if (state.exit_rcu)
 			rcu_irq_exit();
 	}
+
+out:
+	if (synchronized)
+		irqentry_unstall();
+
+	return;
 }
 
 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
diff --git a/kernel/kernel/exit.c b/kernel/kernel/exit.c
index 86e403191d..9150d666b4 100644
--- a/kernel/kernel/exit.c
+++ b/kernel/kernel/exit.c
@@ -14,6 +14,7 @@
 #include <linux/sched/task_stack.h>
 #include <linux/sched/cputime.h>
 #include <linux/interrupt.h>
+#include <linux/irq_pipeline.h>
 #include <linux/module.h>
 #include <linux/capability.h>
 #include <linux/completion.h>
@@ -767,6 +768,7 @@ void __noreturn do_exit(long code)
 
 	io_uring_files_cancel();
 	exit_signals(tsk);  /* sets PF_EXITING */
+	inband_exit_notify();
 
 	/* sync mm's RSS info before statistics gathering */
 	if (tsk->mm)
diff --git a/kernel/kernel/fork.c b/kernel/kernel/fork.c
index f73e3e694a..978ecbf5af 100644
--- a/kernel/kernel/fork.c
+++ b/kernel/kernel/fork.c
@@ -48,6 +48,7 @@
 #include <linux/cpu.h>
 #include <linux/cgroup.h>
 #include <linux/security.h>
+#include <linux/dovetail.h>
 #include <linux/hugetlb.h>
 #include <linux/seccomp.h>
 #include <linux/swap.h>
@@ -938,6 +939,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 #endif
 
 	setup_thread_stack(tsk, orig);
+	inband_task_init(tsk);
 	clear_user_return_notifier(tsk);
 	clear_tsk_need_resched(tsk);
 	set_task_stack_end_magic(tsk);
@@ -1083,6 +1085,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 #endif
 	mm_init_uprobes_state(mm);
 	hugetlb_count_init(mm);
+#ifdef CONFIG_DOVETAIL
+	memset(&mm->oob_state, 0, sizeof(mm->oob_state));
+#endif
 
 	if (current->mm) {
 		mm->flags = current->mm->flags & MMF_INIT_MASK;
@@ -1131,6 +1136,7 @@ static inline void __mmput(struct mm_struct *mm)
 	exit_aio(mm);
 	ksm_exit(mm);
 	khugepaged_exit(mm); /* must run before exit_mmap */
+	inband_cleanup_notify(mm); /* ditto. */
 	exit_mmap(mm);
 	mm_put_huge_zero_page(mm);
 	set_mm_exe_file(mm, NULL);
diff --git a/kernel/kernel/irq/Kconfig b/kernel/kernel/irq/Kconfig
index 1bd144ed25..d170936eab 100644
--- a/kernel/kernel/irq/Kconfig
+++ b/kernel/kernel/irq/Kconfig
@@ -142,6 +142,19 @@ config GENERIC_IRQ_DEBUGFS
 
 	  If you don't know what to do here, say N.
 
+# Interrupt pipeline
+config HAVE_IRQ_PIPELINE
+	bool
+
+config IRQ_PIPELINE
+	bool "Interrupt pipeline"
+	depends on HAVE_IRQ_PIPELINE
+	select IRQ_DOMAIN
+	default n
+	help
+	  Activate this option if you want the interrupt pipeline to be
+	  compiled in.
+
 endmenu
 
 config GENERIC_IRQ_MULTI_HANDLER
diff --git a/kernel/kernel/irq/Makefile b/kernel/kernel/irq/Makefile
index b4f53717d1..b6e43ec9b2 100644
--- a/kernel/kernel/irq/Makefile
+++ b/kernel/kernel/irq/Makefile
@@ -9,6 +9,8 @@ obj-$(CONFIG_GENERIC_IRQ_CHIP) += generic-chip.o
 obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o
 obj-$(CONFIG_IRQ_DOMAIN) += irqdomain.o
 obj-$(CONFIG_IRQ_SIM) += irq_sim.o
+obj-$(CONFIG_IRQ_PIPELINE) += pipeline.o
+obj-$(CONFIG_IRQ_PIPELINE_TORTURE_TEST) += irqptorture.o
 obj-$(CONFIG_PROC_FS) += proc.o
 obj-$(CONFIG_GENERIC_PENDING_IRQ) += migration.o
 obj-$(CONFIG_GENERIC_IRQ_MIGRATION) += cpuhotplug.o
diff --git a/kernel/kernel/irq/chip.c b/kernel/kernel/irq/chip.c
index 520b9fa620..13edfa8a92 100644
--- a/kernel/kernel/irq/chip.c
+++ b/kernel/kernel/irq/chip.c
@@ -15,6 +15,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/irqdomain.h>
 #include <linux/wakeup_reason.h>
+#include <linux/irq_pipeline.h>
 
 #include <trace/events/irq.h>
 
@@ -49,6 +50,10 @@ int irq_set_chip(unsigned int irq, struct irq_chip *chip)
 
 	if (!chip)
 		chip = &no_irq_chip;
+	else
+		WARN_ONCE(irqs_pipelined() &&
+			  (chip->flags & IRQCHIP_PIPELINE_SAFE) == 0,
+			  "irqchip %s is not pipeline-safe!", chip->name);
 
 	desc->irq_data.chip = chip;
 	irq_put_desc_unlock(desc, flags);
@@ -156,14 +161,6 @@ int irq_set_chip_data(unsigned int irq, void *data)
 }
 EXPORT_SYMBOL(irq_set_chip_data);
 
-struct irq_data *irq_get_irq_data(unsigned int irq)
-{
-	struct irq_desc *desc = irq_to_desc(irq);
-
-	return desc ? &desc->irq_data : NULL;
-}
-EXPORT_SYMBOL_GPL(irq_get_irq_data);
-
 static void irq_state_clr_disabled(struct irq_desc *desc)
 {
 	irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
@@ -386,7 +383,8 @@ static void __irq_disable(struct irq_desc *desc, bool mask)
  */
 void irq_disable(struct irq_desc *desc)
 {
-	__irq_disable(desc, irq_settings_disable_unlazy(desc));
+	__irq_disable(desc,
+	      irq_settings_disable_unlazy(desc) || irqs_pipelined());
 }
 
 void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
@@ -532,8 +530,22 @@ static bool irq_may_run(struct irq_desc *desc)
 	 * If the interrupt is an armed wakeup source, mark it pending
 	 * and suspended, disable it and notify the pm core about the
 	 * event.
+	 *
+	 * When pipelining, the logic is as follows:
+	 *
+	 * - from a pipeline entry context, we might have preempted
+	 * the oob stage, or irqs might be [virtually] off, so we may
+	 * not run the in-band PM code. Just make sure any wakeup
+	 * interrupt is detected later on when the flow handler
+	 * re-runs from the in-band stage.
+	 *
+	 * - from the in-band context, run the PM wakeup check.
 	 */
-	if (irq_pm_check_wakeup(desc))
+	if (irqs_pipelined()) {
+		WARN_ON_ONCE(irq_pipeline_debug() && !in_pipeline());
+		if (irqd_is_wakeup_armed(&desc->irq_data))
+			return true;
+	} else if (irq_pm_check_wakeup(desc))
 		return false;
 
 	/*
@@ -557,8 +569,13 @@ void handle_simple_irq(struct irq_desc *desc)
 {
 	raw_spin_lock(&desc->lock);
 
-	if (!irq_may_run(desc))
+	if (start_irq_flow() && !irq_may_run(desc))
+		goto out_unlock;
+
+	if (on_pipeline_entry()) {
+		handle_oob_irq(desc);
 		goto out_unlock;
+	}
 
 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
 
@@ -594,9 +611,14 @@ void handle_untracked_irq(struct irq_desc *desc)
 
 	raw_spin_lock(&desc->lock);
 
-	if (!irq_may_run(desc))
+	if (start_irq_flow() && !irq_may_run(desc))
 		goto out_unlock;
 
+	if (on_pipeline_entry()) {
+		handle_oob_irq(desc);
+		goto out_unlock;
+	}
+
 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
 
 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
@@ -618,6 +640,20 @@ void handle_untracked_irq(struct irq_desc *desc)
 }
 EXPORT_SYMBOL_GPL(handle_untracked_irq);
 
+static inline void cond_eoi_irq(struct irq_desc *desc)
+{
+	struct irq_chip *chip = desc->irq_data.chip;
+
+	if (!(chip->flags & IRQCHIP_EOI_THREADED))
+		chip->irq_eoi(&desc->irq_data);
+}
+
+static inline void mask_cond_eoi_irq(struct irq_desc *desc)
+{
+	mask_irq(desc);
+	cond_eoi_irq(desc);
+}
+
 /*
  * Called unconditionally from handle_level_irq() and only for oneshot
  * interrupts from handle_fasteoi_irq()
@@ -648,10 +684,19 @@ static void cond_unmask_irq(struct irq_desc *desc)
 void handle_level_irq(struct irq_desc *desc)
 {
 	raw_spin_lock(&desc->lock);
-	mask_ack_irq(desc);
 
-	if (!irq_may_run(desc))
+	if (start_irq_flow()) {
+		mask_ack_irq(desc);
+
+		if (!irq_may_run(desc))
+			goto out_unlock;
+	}
+
+	if (on_pipeline_entry()) {
+		if (handle_oob_irq(desc))
+			goto out_unmask;
 		goto out_unlock;
+	}
 
 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
 
@@ -666,7 +711,7 @@ void handle_level_irq(struct irq_desc *desc)
 
 	kstat_incr_irqs_this_cpu(desc);
 	handle_irq_event(desc);
-
+out_unmask:
 	cond_unmask_irq(desc);
 
 out_unlock:
@@ -677,7 +722,10 @@ EXPORT_SYMBOL_GPL(handle_level_irq);
 static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
 {
 	if (!(desc->istate & IRQS_ONESHOT)) {
-		chip->irq_eoi(&desc->irq_data);
+		if (!irqs_pipelined())
+			chip->irq_eoi(&desc->irq_data);
+		else if (!irqd_irq_disabled(&desc->irq_data))
+			unmask_irq(desc);
 		return;
 	}
 	/*
@@ -688,9 +736,11 @@ static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
 	 */
 	if (!irqd_irq_disabled(&desc->irq_data) &&
 	    irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
-		chip->irq_eoi(&desc->irq_data);
+		if (!irqs_pipelined())
+			chip->irq_eoi(&desc->irq_data);
 		unmask_irq(desc);
-	} else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
+	} else if (!irqs_pipelined() &&
+		   !(chip->flags & IRQCHIP_EOI_THREADED)) {
 		chip->irq_eoi(&desc->irq_data);
 	}
 }
@@ -710,9 +760,17 @@ void handle_fasteoi_irq(struct irq_desc *desc)
 
 	raw_spin_lock(&desc->lock);
 
-	if (!irq_may_run(desc))
+	if (start_irq_flow() && !irq_may_run(desc))
 		goto out;
 
+	if (on_pipeline_entry()) {
+		if (handle_oob_irq(desc))
+			chip->irq_eoi(&desc->irq_data);
+		else
+			mask_cond_eoi_irq(desc);
+		goto out_unlock;
+	}
+
 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
 
 	/*
@@ -726,13 +784,13 @@ void handle_fasteoi_irq(struct irq_desc *desc)
 	}
 
 	kstat_incr_irqs_this_cpu(desc);
-	if (desc->istate & IRQS_ONESHOT)
+	if (!irqs_pipelined() && (desc->istate & IRQS_ONESHOT))
 		mask_irq(desc);
 
 	handle_irq_event(desc);
 
 	cond_unmask_eoi_irq(desc, chip);
-
+out_unlock:
 	raw_spin_unlock(&desc->lock);
 	return;
 out:
@@ -792,30 +850,42 @@ EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
  */
 void handle_edge_irq(struct irq_desc *desc)
 {
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+
 	raw_spin_lock(&desc->lock);
 
-	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
+	if (start_irq_flow()) {
+		desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
 
-	if (!irq_may_run(desc)) {
-		desc->istate |= IRQS_PENDING;
-		mask_ack_irq(desc);
-		goto out_unlock;
+		if (!irq_may_run(desc)) {
+			desc->istate |= IRQS_PENDING;
+			mask_ack_irq(desc);
+			goto out_unlock;
+		}
+
+		/*
+		 * If its disabled or no action available then mask it
+		 * and get out of here.
+		 */
+		if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
+			desc->istate |= IRQS_PENDING;
+			mask_ack_irq(desc);
+			goto out_unlock;
+		}
 	}
 
-	/*
-	 * If its disabled or no action available then mask it and get
-	 * out of here.
-	 */
-	if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
-		desc->istate |= IRQS_PENDING;
-		mask_ack_irq(desc);
+	if (on_pipeline_entry()) {
+		chip->irq_ack(&desc->irq_data);
+		desc->istate |= IRQS_EDGE;
+		handle_oob_irq(desc);
 		goto out_unlock;
 	}
 
 	kstat_incr_irqs_this_cpu(desc);
 
 	/* Start handling the irq */
-	desc->irq_data.chip->irq_ack(&desc->irq_data);
+	if (!irqs_pipelined())
+		chip->irq_ack(&desc->irq_data);
 
 	do {
 		if (unlikely(!desc->action)) {
@@ -840,6 +910,8 @@ void handle_edge_irq(struct irq_desc *desc)
 		 !irqd_irq_disabled(&desc->irq_data));
 
 out_unlock:
+	if (on_pipeline_entry())
+		desc->istate &= ~IRQS_EDGE;
 	raw_spin_unlock(&desc->lock);
 }
 EXPORT_SYMBOL(handle_edge_irq);
@@ -858,11 +930,20 @@ void handle_edge_eoi_irq(struct irq_desc *desc)
 
 	raw_spin_lock(&desc->lock);
 
-	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
+	if (start_irq_flow()) {
+		desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
 
-	if (!irq_may_run(desc)) {
-		desc->istate |= IRQS_PENDING;
-		goto out_eoi;
+		if (!irq_may_run(desc)) {
+			desc->istate |= IRQS_PENDING;
+			goto out_eoi;
+		}
+	}
+
+	if (on_pipeline_entry()) {
+		desc->istate |= IRQS_EDGE;
+		if (handle_oob_irq(desc))
+			goto out_eoi;
+		goto out;
 	}
 
 	/*
@@ -887,6 +968,9 @@ void handle_edge_eoi_irq(struct irq_desc *desc)
 
 out_eoi:
 	chip->irq_eoi(&desc->irq_data);
+out:
+	if (on_pipeline_entry())
+		desc->istate &= ~IRQS_EDGE;
 	raw_spin_unlock(&desc->lock);
 }
 #endif
@@ -900,6 +984,18 @@ void handle_edge_eoi_irq(struct irq_desc *desc)
 void handle_percpu_irq(struct irq_desc *desc)
 {
 	struct irq_chip *chip = irq_desc_get_chip(desc);
+	bool handled;
+
+	if (on_pipeline_entry()) {
+		if (chip->irq_ack)
+			chip->irq_ack(&desc->irq_data);
+		handled = handle_oob_irq(desc);
+		if (chip->irq_eoi)
+			chip->irq_eoi(&desc->irq_data);
+		if (!handled && chip->irq_mask)
+			chip->irq_mask(&desc->irq_data);
+		return;
+	}
 
 	/*
 	 * PER CPU interrupts are not serialized. Do not touch
@@ -907,13 +1003,17 @@ void handle_percpu_irq(struct irq_desc *desc)
 	 */
 	__kstat_incr_irqs_this_cpu(desc);
 
-	if (chip->irq_ack)
-		chip->irq_ack(&desc->irq_data);
-
-	handle_irq_event_percpu(desc);
-
-	if (chip->irq_eoi)
-		chip->irq_eoi(&desc->irq_data);
+	if (irqs_pipelined()) {
+		handle_irq_event_percpu(desc);
+		if (chip->irq_unmask)
+			chip->irq_unmask(&desc->irq_data);
+	} else {
+		if (chip->irq_ack)
+			chip->irq_ack(&desc->irq_data);
+		handle_irq_event_percpu(desc);
+		if (chip->irq_eoi)
+			chip->irq_eoi(&desc->irq_data);
+	}
 }
 
 /**
@@ -933,6 +1033,18 @@ void handle_percpu_devid_irq(struct irq_desc *desc)
 	struct irqaction *action = desc->action;
 	unsigned int irq = irq_desc_get_irq(desc);
 	irqreturn_t res;
+	bool handled;
+
+	if (on_pipeline_entry()) {
+		if (chip->irq_ack)
+			chip->irq_ack(&desc->irq_data);
+		handled = handle_oob_irq(desc);
+		if (chip->irq_eoi)
+			chip->irq_eoi(&desc->irq_data);
+		if (!handled && chip->irq_mask)
+			chip->irq_mask(&desc->irq_data);
+		return;
+	}
 
 	/*
 	 * PER CPU interrupts are not serialized. Do not touch
@@ -940,7 +1052,7 @@ void handle_percpu_devid_irq(struct irq_desc *desc)
 	 */
 	__kstat_incr_irqs_this_cpu(desc);
 
-	if (chip->irq_ack)
+	if (!irqs_pipelined() && chip->irq_ack)
 		chip->irq_ack(&desc->irq_data);
 
 	if (likely(action)) {
@@ -958,8 +1070,11 @@ void handle_percpu_devid_irq(struct irq_desc *desc)
 			    enabled ? " and unmasked" : "", irq, cpu);
 	}
 
-	if (chip->irq_eoi)
-		chip->irq_eoi(&desc->irq_data);
+	if (irqs_pipelined()) {
+		if (chip->irq_unmask)
+			chip->irq_unmask(&desc->irq_data);
+	} else if (chip->irq_eoi)
+			chip->irq_eoi(&desc->irq_data);
 }
 
 /**
@@ -979,11 +1094,22 @@ void handle_percpu_devid_fasteoi_ipi(struct irq_desc *desc)
 	unsigned int irq = irq_desc_get_irq(desc);
 	irqreturn_t res;
 
-	__kstat_incr_irqs_this_cpu(desc);
-
 	if (chip->irq_eoi)
 		chip->irq_eoi(&desc->irq_data);
 
+	if (on_pipeline_entry()) {
+		handle_oob_irq(desc);
+		return;
+	}
+
+	/* Trap spurious IPIs if pipelined. */
+	if (irqs_pipelined() && !action) {
+		print_irq_desc(irq, desc);
+		return;
+	}
+
+	__kstat_incr_irqs_this_cpu(desc);
+
 	trace_irq_handler_entry(irq, action);
 	res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
 	trace_irq_handler_exit(irq, action, res);
@@ -1076,6 +1202,7 @@ __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
 			desc->handle_irq = handle;
 		}
 
+		irq_settings_set_chained(desc);
 		irq_settings_set_noprobe(desc);
 		irq_settings_set_norequest(desc);
 		irq_settings_set_nothread(desc);
@@ -1251,9 +1378,18 @@ void handle_fasteoi_ack_irq(struct irq_desc *desc)
 
 	raw_spin_lock(&desc->lock);
 
-	if (!irq_may_run(desc))
+	if (start_irq_flow() && !irq_may_run(desc))
 		goto out;
 
+	if (on_pipeline_entry()) {
+		chip->irq_ack(&desc->irq_data);
+		if (handle_oob_irq(desc))
+			chip->irq_eoi(&desc->irq_data);
+		else
+			mask_cond_eoi_irq(desc);
+		goto out_unlock;
+	}
+
 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
 
 	/*
@@ -1267,11 +1403,13 @@ void handle_fasteoi_ack_irq(struct irq_desc *desc)
 	}
 
 	kstat_incr_irqs_this_cpu(desc);
-	if (desc->istate & IRQS_ONESHOT)
-		mask_irq(desc);
+	if (!irqs_pipelined()) {
+		if (desc->istate & IRQS_ONESHOT)
+			mask_irq(desc);
 
-	/* Start handling the irq */
-	desc->irq_data.chip->irq_ack(&desc->irq_data);
+		/* Start handling the irq */
+		chip->irq_ack(&desc->irq_data);
+	}
 
 	handle_irq_event(desc);
 
@@ -1282,6 +1420,7 @@ void handle_fasteoi_ack_irq(struct irq_desc *desc)
 out:
 	if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
 		chip->irq_eoi(&desc->irq_data);
+out_unlock:
 	raw_spin_unlock(&desc->lock);
 }
 EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
@@ -1301,10 +1440,21 @@ void handle_fasteoi_mask_irq(struct irq_desc *desc)
 	struct irq_chip *chip = desc->irq_data.chip;
 
 	raw_spin_lock(&desc->lock);
-	mask_ack_irq(desc);
 
-	if (!irq_may_run(desc))
-		goto out;
+	if (start_irq_flow()) {
+		mask_ack_irq(desc);
+
+		if (!irq_may_run(desc))
+			goto out;
+	}
+
+	if (on_pipeline_entry()) {
+		if (handle_oob_irq(desc))
+			chip->irq_eoi(&desc->irq_data);
+		else
+			cond_eoi_irq(desc);
+		goto out_unlock;
+	}
 
 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
 
@@ -1319,7 +1469,7 @@ void handle_fasteoi_mask_irq(struct irq_desc *desc)
 	}
 
 	kstat_incr_irqs_this_cpu(desc);
-	if (desc->istate & IRQS_ONESHOT)
+	if (!irqs_pipelined() && (desc->istate & IRQS_ONESHOT))
 		mask_irq(desc);
 
 	handle_irq_event(desc);
@@ -1331,6 +1481,7 @@ void handle_fasteoi_mask_irq(struct irq_desc *desc)
 out:
 	if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
 		chip->irq_eoi(&desc->irq_data);
+out_unlock:
 	raw_spin_unlock(&desc->lock);
 }
 EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
diff --git a/kernel/kernel/irq/cpuhotplug.c b/kernel/kernel/irq/cpuhotplug.c
index cf8d4f7563..867db10e1b 100644
--- a/kernel/kernel/irq/cpuhotplug.c
+++ b/kernel/kernel/irq/cpuhotplug.c
@@ -156,6 +156,9 @@ void irq_migrate_all_off_this_cpu(void)
 {
 	struct irq_desc *desc;
 	unsigned int irq;
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
 
 	for_each_active_irq(irq) {
 		bool affinity_broken;
@@ -170,6 +173,8 @@ void irq_migrate_all_off_this_cpu(void)
 					    irq, smp_processor_id());
 		}
 	}
+
+	hard_local_irq_restore(flags);
 }
 
 static bool hk_should_isolate(struct irq_data *data, unsigned int cpu)
diff --git a/kernel/kernel/irq/debug.h b/kernel/kernel/irq/debug.h
index 8ccb326d29..40f7268457 100644
--- a/kernel/kernel/irq/debug.h
+++ b/kernel/kernel/irq/debug.h
@@ -33,6 +33,8 @@ static inline void print_irq_desc(unsigned int irq, struct irq_desc *desc)
 	___P(IRQ_NOREQUEST);
 	___P(IRQ_NOTHREAD);
 	___P(IRQ_NOAUTOEN);
+	___P(IRQ_OOB);
+	___P(IRQ_CHAINED);
 
 	___PS(IRQS_AUTODETECT);
 	___PS(IRQS_REPLAY);
diff --git a/kernel/kernel/irq/dummychip.c b/kernel/kernel/irq/dummychip.c
index 0b0cdf206d..7bf8cbee1b 100644
--- a/kernel/kernel/irq/dummychip.c
+++ b/kernel/kernel/irq/dummychip.c
@@ -43,7 +43,7 @@ struct irq_chip no_irq_chip = {
 	.irq_enable	= noop,
 	.irq_disable	= noop,
 	.irq_ack	= ack_bad,
-	.flags		= IRQCHIP_SKIP_SET_WAKE,
+	.flags		= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_PIPELINE_SAFE,
 };
 
 /*
@@ -59,6 +59,6 @@ struct irq_chip dummy_irq_chip = {
 	.irq_ack	= noop,
 	.irq_mask	= noop,
 	.irq_unmask	= noop,
-	.flags		= IRQCHIP_SKIP_SET_WAKE,
+	.flags		= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_PIPELINE_SAFE,
 };
 EXPORT_SYMBOL_GPL(dummy_irq_chip);
diff --git a/kernel/kernel/irq/generic-chip.c b/kernel/kernel/irq/generic-chip.c
index 79cb6d063b..6ca043e6ec 100644
--- a/kernel/kernel/irq/generic-chip.c
+++ b/kernel/kernel/irq/generic-chip.c
@@ -16,7 +16,7 @@
 #include "internals.h"
 
 static LIST_HEAD(gc_list);
-static DEFINE_RAW_SPINLOCK(gc_lock);
+static DEFINE_HARD_SPINLOCK(gc_lock);
 
 /**
  * irq_gc_noop - NOOP function
diff --git a/kernel/kernel/irq/handle.c b/kernel/kernel/irq/handle.c
index 8806444a68..e4fe9c6708 100644
--- a/kernel/kernel/irq/handle.c
+++ b/kernel/kernel/irq/handle.c
@@ -32,9 +32,16 @@ void handle_bad_irq(struct irq_desc *desc)
 {
 	unsigned int irq = irq_desc_get_irq(desc);
 
+	/* Let the in-band stage report the issue. */
+	if (on_pipeline_entry()) {
+		ack_bad_irq(irq);
+		return;
+	}
+
 	print_irq_desc(irq, desc);
 	kstat_incr_irqs_this_cpu(desc);
-	ack_bad_irq(irq);
+	if (!irqs_pipelined())
+		ack_bad_irq(irq);
 }
 EXPORT_SYMBOL_GPL(handle_bad_irq);
 
diff --git a/kernel/kernel/irq/internals.h b/kernel/kernel/irq/internals.h
index e58342ace1..341c8f658e 100644
--- a/kernel/kernel/irq/internals.h
+++ b/kernel/kernel/irq/internals.h
@@ -52,6 +52,7 @@ enum {
  * IRQS_PENDING			- irq is pending and replayed later
  * IRQS_SUSPENDED		- irq is suspended
  * IRQS_NMI			- irq line is used to deliver NMIs
+ * IRQS_EDGE			- irq line received an edge event
  */
 enum {
 	IRQS_AUTODETECT		= 0x00000001,
@@ -64,6 +65,7 @@ enum {
 	IRQS_SUSPENDED		= 0x00000800,
 	IRQS_TIMINGS		= 0x00001000,
 	IRQS_NMI		= 0x00002000,
+	IRQS_EDGE		= 0x00004000,
 };
 
 #include "debug.h"
diff --git a/kernel/kernel/irq/irqdesc.c b/kernel/kernel/irq/irqdesc.c
index 2f35de34fe..846c2c8e52 100644
--- a/kernel/kernel/irq/irqdesc.c
+++ b/kernel/kernel/irq/irqdesc.c
@@ -16,6 +16,7 @@
 #include <linux/bitmap.h>
 #include <linux/irqdomain.h>
 #include <linux/sysfs.h>
+#include <linux/irq_pipeline.h>
 
 #include "internals.h"
 
@@ -453,6 +454,7 @@ static void free_desc(unsigned int irq)
 	 * irq_sysfs_init() as well.
 	 */
 	irq_sysfs_del(desc);
+	uncache_irq_desc(irq);
 	delete_irq_desc(irq);
 
 	/*
@@ -633,9 +635,12 @@ void irq_init_desc(unsigned int irq)
 #endif /* !CONFIG_SPARSE_IRQ */
 
 /**
- * generic_handle_irq - Invoke the handler for a particular irq
+ * generic_handle_irq - Handle a particular irq
  * @irq:	The irq number to handle
  *
+ * The handler is invoked, unless we are entering the interrupt
+ * pipeline, in which case the incoming IRQ is only scheduled for
+ * deferred delivery.
  */
 int generic_handle_irq(unsigned int irq)
 {
@@ -646,7 +651,7 @@ int generic_handle_irq(unsigned int irq)
 		return -EINVAL;
 
 	data = irq_desc_get_irq_data(desc);
-	if (WARN_ON_ONCE(!in_irq() && handle_enforce_irqctx(data)))
+	if (WARN_ON_ONCE(!in_hard_irq() && handle_enforce_irqctx(data)))
 		return -EPERM;
 
 	generic_handle_irq_desc(desc);
diff --git a/kernel/kernel/irq/irqptorture.c b/kernel/kernel/irq/irqptorture.c
new file mode 100644
index 0000000000..2518c47415
--- /dev/null
+++ b/kernel/kernel/irq/irqptorture.c
@@ -0,0 +1,254 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2017 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#include <linux/kernel.h>
+#include <linux/torture.h>
+#include <linux/printk.h>
+#include <linux/delay.h>
+#include <linux/tick.h>
+#include <linux/smp.h>
+#include <linux/cpumask.h>
+#include <linux/clockchips.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irq_pipeline.h>
+#include <linux/stop_machine.h>
+#include <linux/irq_work.h>
+#include <linux/completion.h>
+#include <linux/slab.h>
+#include "settings.h"
+
+static void torture_event_handler(struct clock_event_device *dev)
+{
+	/*
+	 * We are running on the oob stage, in NMI-like mode. Schedule
+	 * a tick on the proxy device to satisfy the corresponding
+	 * timing request asap.
+	 */
+	tick_notify_proxy();
+}
+
+static void setup_proxy(struct clock_proxy_device *dev)
+{
+	dev->handle_oob_event = torture_event_handler;
+}
+
+static int start_tick_takeover_test(void)
+{
+	return tick_install_proxy(setup_proxy, cpu_online_mask);
+}
+
+static void stop_tick_takeover_test(void)
+{
+	tick_uninstall_proxy(cpu_online_mask);
+}
+
+struct stop_machine_p_data {
+	int origin_cpu;
+	cpumask_var_t disable_mask;
+};
+
+static int stop_machine_handler(void *arg)
+{
+	struct stop_machine_p_data *p = arg;
+	int cpu = raw_smp_processor_id();
+
+	/*
+	 * The stop_machine() handler must run with hard
+	 * IRQs off, note the current state in the result mask.
+	 */
+	if (hard_irqs_disabled())
+		cpumask_set_cpu(cpu, p->disable_mask);
+
+	if (cpu != p->origin_cpu)
+		pr_alert("irq_pipeline" TORTURE_FLAG
+			 " CPU%d responds to stop_machine()\n", cpu);
+	return 0;
+}
+
+/*
+ * We test stop_machine() as a way to validate IPI handling in a
+ * pipelined interrupt context.
+ */
+static int test_stop_machine(void)
+{
+	struct stop_machine_p_data d;
+	cpumask_var_t tmp_mask;
+	int ret = -ENOMEM, cpu;
+
+	if (!zalloc_cpumask_var(&d.disable_mask, GFP_KERNEL)) {
+		WARN_ON(1);
+		return ret;
+	}
+
+	if (!alloc_cpumask_var(&tmp_mask, GFP_KERNEL)) {
+		WARN_ON(1);
+		goto fail;
+	}
+
+	ret = -EINVAL;
+	d.origin_cpu = raw_smp_processor_id();
+	pr_alert("irq_pipeline" TORTURE_FLAG
+		 " CPU%d initiates stop_machine()\n",
+		 d.origin_cpu);
+
+	ret = stop_machine(stop_machine_handler, &d, cpu_online_mask);
+	WARN_ON(ret);
+	if (ret)
+		goto fail;
+
+	/*
+	 * Check whether all handlers did run with hard IRQs off. If
+	 * some of them did not, then we have a problem with the stop
+	 * IRQ delivery.
+	 */
+	cpumask_xor(tmp_mask, cpu_online_mask, d.disable_mask);
+	if (!cpumask_empty(tmp_mask)) {
+		for_each_cpu(cpu, tmp_mask)
+			pr_alert("irq_pipeline" TORTURE_FLAG
+				 " CPU%d: hard IRQs ON in stop_machine()"
+				 " handler!\n", cpu);
+	}
+
+	free_cpumask_var(tmp_mask);
+fail:
+	free_cpumask_var(d.disable_mask);
+
+	return ret;
+}
+
+static struct irq_work_tester {
+	struct irq_work work;
+	struct completion done;
+} irq_work_tester;
+
+static void irq_work_handler(struct irq_work *work)
+{
+	int cpu = raw_smp_processor_id();
+
+	if (!running_inband()) {
+		pr_alert("irq_pipeline" TORTURE_FLAG
+			 " CPU%d: irq_work handler not running on"
+			 " in-band stage?!\n", cpu);
+		return;
+	}
+
+	if (work != &irq_work_tester.work)
+		pr_alert("irq_pipeline" TORTURE_FLAG
+			 " CPU%d: irq_work handler received broken"
+			 " arg?!\n", cpu);
+	else {
+		pr_alert("irq_pipeline" TORTURE_FLAG
+			 " CPU%d: irq_work handled\n", cpu);
+		complete(&irq_work_tester.done);
+	}
+}
+
+static int trigger_oob_work(void *arg)
+{
+	int cpu = raw_smp_processor_id();
+
+	if (!running_oob()) {
+		pr_alert("irq_pipeline" TORTURE_FLAG
+			 " CPU%d: escalated request not running on"
+			 " oob stage?!\n", cpu);
+		return -EINVAL;
+	}
+
+	if ((struct irq_work_tester *)arg != &irq_work_tester) {
+		pr_alert("irq_pipeline" TORTURE_FLAG
+			 " CPU%d: escalation handler received broken"
+			 " arg?!\n", cpu);
+		return -EINVAL;
+	}
+
+	irq_work_queue(&irq_work_tester.work);
+	pr_alert("irq_pipeline" TORTURE_FLAG
+		 " CPU%d: stage escalation request works\n",
+		 cpu);
+
+	return 0;
+}
+
+static int test_interstage_work_injection(void)
+{
+	struct irq_work_tester *p = &irq_work_tester;
+	int ret, cpu = raw_smp_processor_id();
+	unsigned long rem;
+
+	init_completion(&p->done);
+	init_irq_work(&p->work, irq_work_handler);
+
+	/* Trigger over the in-band stage. */
+	irq_work_queue(&p->work);
+	rem = wait_for_completion_timeout(&p->done, HZ / 10);
+	if (!rem) {
+		pr_alert("irq_pipeline" TORTURE_FLAG
+			 " CPU%d: irq_work trigger from in-band stage not handled!\n",
+			 cpu);
+		return -EINVAL;
+	}
+
+	pr_alert("irq_pipeline" TORTURE_FLAG
+		 " CPU%d: in-band->in-band irq_work trigger works\n", cpu);
+
+	reinit_completion(&p->done);
+
+	/* Now try over the oob stage. */
+	ret = run_oob_call(trigger_oob_work, p);
+	if (ret)
+		return ret;
+
+	ret = wait_for_completion_timeout(&p->done, HZ / 10);
+	if (!rem) {
+		pr_alert("irq_pipeline" TORTURE_FLAG
+			 " CPU%d: irq_work trigger from oob"
+			 " stage not handled!\n", cpu);
+		return -EINVAL;
+	}
+
+	pr_alert("irq_pipeline" TORTURE_FLAG
+		 " CPU%d: oob->in-band irq_work trigger works\n",
+		 cpu);
+
+	return 0;
+}
+
+static int __init irqp_torture_init(void)
+{
+	int ret;
+
+	pr_info("Starting IRQ pipeline tests...");
+
+	ret = enable_oob_stage("torture");
+	if (ret) {
+		if (ret == -EBUSY)
+			pr_alert("irq_pipeline" TORTURE_FLAG
+			 " won't run, oob stage '%s' is already installed",
+			 oob_stage.name);
+
+		return ret;
+	}
+
+	ret = test_stop_machine();
+	if (ret)
+		goto out;
+
+	ret = start_tick_takeover_test();
+	if (ret)
+		goto out;
+
+	ret = test_interstage_work_injection();
+	if (!ret)
+		msleep(1000);
+
+	stop_tick_takeover_test();
+out:
+	disable_oob_stage();
+	pr_info("IRQ pipeline tests %s.", ret ? "FAILED" : "OK");
+
+	return 0;
+}
+late_initcall(irqp_torture_init);
diff --git a/kernel/kernel/irq/manage.c b/kernel/kernel/irq/manage.c
index 76da8deea3..4b06f5a919 100644
--- a/kernel/kernel/irq/manage.c
+++ b/kernel/kernel/irq/manage.c
@@ -10,6 +10,7 @@
 
 #include <linux/irq.h>
 #include <linux/kthread.h>
+#include <linux/kconfig.h>
 #include <linux/module.h>
 #include <linux/random.h>
 #include <linux/interrupt.h>
@@ -829,6 +830,50 @@ int irq_set_irq_wake(unsigned int irq, unsigned int on)
 }
 EXPORT_SYMBOL(irq_set_irq_wake);
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+/**
+ *	irq_switch_oob - Control out-of-band setting for a registered IRQ descriptor
+ *	@irq:	interrupt to control
+ *	@on:	enable/disable pipelining
+ *
+ *	Enable/disable out-of-band handling for an IRQ. At least one
+ *	action must have been previously registered for such
+ *	interrupt.
+ *
+ *      The previously registered action(s) need(s) not bearing the
+ *      IRQF_OOB flag for the IRQ to be switched to out-of-band
+ *      handling. This call enables switching pre-installed IRQs from
+ *      in-band to out-of-band handling.
+ *
+ *      NOTE: This routine affects all action handlers sharing the
+ *      IRQ.
+ */
+int irq_switch_oob(unsigned int irq, bool on)
+{
+	struct irq_desc *desc;
+	unsigned long flags;
+	int ret = 0;
+
+	desc = irq_get_desc_lock(irq, &flags, 0);
+	if (!desc)
+		return -EINVAL;
+
+	if (!desc->action)
+		ret = -EINVAL;
+	else if (on)
+		irq_settings_set_oob(desc);
+	else
+		irq_settings_clr_oob(desc);
+
+	irq_put_desc_unlock(desc, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(irq_switch_oob);
+
+#endif /* CONFIG_IRQ_PIPELINE */
+
 /*
  * Internal function that tells the architecture code whether a
  * particular irq has been exclusively allocated or is available
@@ -845,7 +890,8 @@ int can_request_irq(unsigned int irq, unsigned long irqflags)
 
 	if (irq_settings_can_request(desc)) {
 		if (!desc->action ||
-		    irqflags & desc->action->flags & IRQF_SHARED)
+		    ((irqflags & desc->action->flags & IRQF_SHARED) &&
+		     !((irqflags ^ desc->action->flags) & IRQF_OOB)))
 			canrequest = 1;
 	}
 	irq_put_desc_unlock(desc, flags);
@@ -1419,6 +1465,21 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 
 	new->irq = irq;
 
+	ret = -EINVAL;
+	/*
+	 *  Out-of-band interrupts can be shared but not threaded.  We
+	 *  silently ignore the OOB setting if interrupt pipelining is
+	 *  disabled.
+	 */
+	if (!irqs_pipelined())
+		new->flags &= ~IRQF_OOB;
+	else if (new->flags & IRQF_OOB) {
+		if (new->thread_fn)
+			goto out_mput;
+		new->flags |= IRQF_NO_THREAD;
+		new->flags &= ~IRQF_ONESHOT;
+	}
+
 	/*
 	 * If the trigger type is not specified by the caller,
 	 * then use the default for this interrupt.
@@ -1432,10 +1493,8 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 	 */
 	nested = irq_settings_is_nested_thread(desc);
 	if (nested) {
-		if (!new->thread_fn) {
-			ret = -EINVAL;
+		if (!new->thread_fn)
 			goto out_mput;
-		}
 		/*
 		 * Replace the primary handler which was provided from
 		 * the driver for non nested interrupt handling by the
@@ -1519,7 +1578,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 		 * the same type (level, edge, polarity). So both flag
 		 * fields must have IRQF_SHARED set and the bits which
 		 * set the trigger type must match. Also all must
-		 * agree on ONESHOT.
+		 * agree on ONESHOT and OOB.
 		 * Interrupt lines used for NMIs cannot be shared.
 		 */
 		unsigned int oldtype;
@@ -1544,7 +1603,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 
 		if (!((old->flags & new->flags) & IRQF_SHARED) ||
 		    (oldtype != (new->flags & IRQF_TRIGGER_MASK)) ||
-		    ((old->flags ^ new->flags) & IRQF_ONESHOT))
+		    ((old->flags ^ new->flags) & (IRQF_OOB|IRQF_ONESHOT)))
 			goto mismatch;
 
 		/* All handlers must agree on per-cpuness */
@@ -1662,6 +1721,9 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 		if (new->flags & IRQF_ONESHOT)
 			desc->istate |= IRQS_ONESHOT;
 
+		if (new->flags & IRQF_OOB)
+			irq_settings_set_oob(desc);
+
 		/* Exclude IRQ from balancing if requested */
 		if (new->flags & IRQF_NOBALANCING) {
 			irq_settings_set_no_balancing(desc);
@@ -1809,6 +1871,8 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
 		irq_settings_clr_disable_unlazy(desc);
 		/* Only shutdown. Deactivate after synchronize_hardirq() */
 		irq_shutdown(desc);
+		/* Turn off OOB handling (after shutdown). */
+		irq_settings_clr_oob(desc);
 	}
 
 #ifdef CONFIG_SMP
@@ -1845,14 +1909,15 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
 
 #ifdef CONFIG_DEBUG_SHIRQ
 	/*
-	 * It's a shared IRQ -- the driver ought to be prepared for an IRQ
-	 * event to happen even now it's being freed, so let's make sure that
-	 * is so by doing an extra call to the handler ....
+	 * It's a shared IRQ (with in-band handler) -- the driver
+	 * ought to be prepared for an IRQ event to happen even now
+	 * it's being freed, so let's make sure that is so by doing an
+	 * extra call to the handler ....
 	 *
 	 * ( We do this after actually deregistering it, to make sure that a
 	 *   'real' IRQ doesn't run in parallel with our fake. )
 	 */
-	if (action->flags & IRQF_SHARED) {
+	if ((action->flags & (IRQF_SHARED|IRQF_OOB)) == IRQF_SHARED) {
 		local_irq_save(flags);
 		action->handler(irq, dev_id);
 		local_irq_restore(flags);
@@ -2473,7 +2538,7 @@ int setup_percpu_irq(unsigned int irq, struct irqaction *act)
  *	__request_percpu_irq - allocate a percpu interrupt line
  *	@irq: Interrupt line to allocate
  *	@handler: Function to be called when the IRQ occurs.
- *	@flags: Interrupt type flags (IRQF_TIMER only)
+ *	@flags: Interrupt type flags (IRQF_TIMER and/or IRQF_OOB only)
  *	@devname: An ascii name for the claiming device
  *	@dev_id: A percpu cookie passed back to the handler function
  *
@@ -2502,7 +2567,7 @@ int __request_percpu_irq(unsigned int irq, irq_handler_t handler,
 	    !irq_settings_is_per_cpu_devid(desc))
 		return -EINVAL;
 
-	if (flags && flags != IRQF_TIMER)
+	if (flags & ~(IRQF_TIMER|IRQF_OOB))
 		return -EINVAL;
 
 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
diff --git a/kernel/kernel/irq/msi.c b/kernel/kernel/irq/msi.c
index b47d95b68a..8fc1d87177 100644
--- a/kernel/kernel/irq/msi.c
+++ b/kernel/kernel/irq/msi.c
@@ -272,6 +272,9 @@ static void msi_domain_update_chip_ops(struct msi_domain_info *info)
 	struct irq_chip *chip = info->chip;
 
 	BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask);
+	WARN_ONCE(IS_ENABLED(CONFIG_IRQ_PIPELINE) &&
+		  (chip->flags & IRQCHIP_PIPELINE_SAFE) == 0,
+		  "MSI domain irqchip %s is not pipeline-safe!", chip->name);
 	if (!chip->irq_set_affinity)
 		chip->irq_set_affinity = msi_domain_set_affinity;
 }
diff --git a/kernel/kernel/irq/pipeline.c b/kernel/kernel/irq/pipeline.c
new file mode 100644
index 0000000000..f64d731e8d
--- /dev/null
+++ b/kernel/kernel/irq/pipeline.c
@@ -0,0 +1,1764 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/irq_pipeline.h>
+#include <linux/irq_work.h>
+#include <linux/jhash.h>
+#include <linux/debug_locks.h>
+#include <linux/dovetail.h>
+#include <dovetail/irq.h>
+#include <trace/events/irq.h>
+#include "internals.h"
+
+#ifdef CONFIG_DEBUG_IRQ_PIPELINE
+#define trace_on_debug
+#else
+#define trace_on_debug  notrace
+#endif
+
+struct irq_stage inband_stage = {
+	.name = "Linux",
+};
+EXPORT_SYMBOL_GPL(inband_stage);
+
+struct irq_stage oob_stage;
+EXPORT_SYMBOL_GPL(oob_stage);
+
+struct irq_domain *synthetic_irq_domain;
+EXPORT_SYMBOL_GPL(synthetic_irq_domain);
+
+bool irq_pipeline_oopsing;
+EXPORT_SYMBOL_GPL(irq_pipeline_oopsing);
+
+bool irq_pipeline_active;
+EXPORT_SYMBOL_GPL(irq_pipeline_active);
+
+#define IRQ_L1_MAPSZ	BITS_PER_LONG
+#define IRQ_L2_MAPSZ	(BITS_PER_LONG * BITS_PER_LONG)
+#define IRQ_FLAT_MAPSZ	DIV_ROUND_UP(IRQ_BITMAP_BITS, BITS_PER_LONG)
+
+#if IRQ_FLAT_MAPSZ > IRQ_L2_MAPSZ
+#define __IRQ_STAGE_MAP_LEVELS	4	/* up to 4/16M vectors */
+#elif IRQ_FLAT_MAPSZ > IRQ_L1_MAPSZ
+#define __IRQ_STAGE_MAP_LEVELS	3	/* up to 64/256M vectors */
+#else
+#define __IRQ_STAGE_MAP_LEVELS	2	/* up to 1024/4096 vectors */
+#endif
+
+struct irq_event_map {
+#if __IRQ_STAGE_MAP_LEVELS >= 3
+	unsigned long index_1[IRQ_L1_MAPSZ];
+#if __IRQ_STAGE_MAP_LEVELS >= 4
+	unsigned long index_2[IRQ_L2_MAPSZ];
+#endif
+#endif
+	unsigned long flat[IRQ_FLAT_MAPSZ];
+};
+
+#ifdef CONFIG_SMP
+
+static struct irq_event_map bootup_irq_map __initdata;
+
+static DEFINE_PER_CPU(struct irq_event_map, irq_map_array[2]);
+
+DEFINE_PER_CPU(struct irq_pipeline_data, irq_pipeline) = {
+	.stages = {
+		[0] = {
+			.log = {
+				.map = &bootup_irq_map,
+			},
+			.stage = &inband_stage,
+		},
+	},
+};
+
+#else /* !CONFIG_SMP */
+
+static struct irq_event_map inband_irq_map;
+
+static struct irq_event_map oob_irq_map;
+
+DEFINE_PER_CPU(struct irq_pipeline_data, irq_pipeline) = {
+	.stages = {
+		[0] = {
+			.log = {
+				.map = &inband_irq_map,
+			},
+			.stage = &inband_stage,
+		},
+		[1] = {
+			.log = {
+				.map = &oob_irq_map,
+			},
+		},
+	},
+};
+
+#endif /* !CONFIG_SMP */
+
+EXPORT_PER_CPU_SYMBOL(irq_pipeline);
+
+static void sirq_noop(struct irq_data *data) { }
+
+/* Virtual interrupt controller for synthetic IRQs. */
+static struct irq_chip sirq_chip = {
+	.name		= "SIRQC",
+	.irq_enable	= sirq_noop,
+	.irq_disable	= sirq_noop,
+	.flags		= IRQCHIP_PIPELINE_SAFE | IRQCHIP_SKIP_SET_WAKE,
+};
+
+static int sirq_map(struct irq_domain *d, unsigned int irq,
+		    irq_hw_number_t hwirq)
+{
+	irq_set_percpu_devid(irq);
+	irq_set_chip_and_handler(irq, &sirq_chip, handle_synthetic_irq);
+
+	return 0;
+}
+
+static struct irq_domain_ops sirq_domain_ops = {
+	.map	= sirq_map,
+};
+
+#ifdef CONFIG_SPARSE_IRQ
+/*
+ * The performances of the radix tree in sparse mode are really ugly
+ * under mm stress on some hw, use a local descriptor cache to ease
+ * the pain.
+ */
+#define DESC_CACHE_SZ  128
+
+static struct irq_desc *desc_cache[DESC_CACHE_SZ] __cacheline_aligned;
+
+static inline u32 hash_irq(unsigned int irq)
+{
+	return jhash(&irq, sizeof(irq), irq) % DESC_CACHE_SZ;
+}
+
+static __always_inline
+struct irq_desc *irq_to_cached_desc(unsigned int irq)
+{
+	int hval = hash_irq(irq);
+	struct irq_desc *desc = desc_cache[hval];
+
+	if (unlikely(desc == NULL || irq_desc_get_irq(desc) != irq)) {
+		desc = irq_to_desc(irq);
+		desc_cache[hval] = desc;
+	}
+
+	return desc;
+}
+
+void uncache_irq_desc(unsigned int irq)
+{
+	int hval = hash_irq(irq);
+
+	desc_cache[hval] = NULL;
+}
+
+#else
+
+static struct irq_desc *irq_to_cached_desc(unsigned int irq)
+{
+	return irq_to_desc(irq);
+}
+
+#endif
+
+/**
+ *	handle_synthetic_irq -  synthetic irq handler
+ *	@desc:	the interrupt description structure for this irq
+ *
+ *	Handles synthetic interrupts flowing down the IRQ pipeline
+ *	with per-CPU semantics.
+ *
+ *      CAUTION: synthetic IRQs may be used to map hardware-generated
+ *      events (e.g. IPIs or traps), we must start handling them as
+ *      common interrupts.
+ */
+void handle_synthetic_irq(struct irq_desc *desc)
+{
+	unsigned int irq = irq_desc_get_irq(desc);
+	struct irqaction *action;
+	irqreturn_t ret;
+	void *dev_id;
+
+	if (on_pipeline_entry()) {
+		handle_oob_irq(desc);
+		return;
+	}
+
+	action = desc->action;
+	if (action == NULL) {
+		if (printk_ratelimit())
+			printk(KERN_WARNING
+			       "CPU%d: WARNING: synthetic IRQ%d has no action.\n",
+			       smp_processor_id(), irq);
+		return;
+	}
+
+	__kstat_incr_irqs_this_cpu(desc);
+	trace_irq_handler_entry(irq, action);
+	dev_id = raw_cpu_ptr(action->percpu_dev_id);
+	ret = action->handler(irq, dev_id);
+	trace_irq_handler_exit(irq, action, ret);
+}
+
+void sync_irq_stage(struct irq_stage *top)
+{
+	struct irq_stage_data *p;
+	struct irq_stage *stage;
+
+	/* We must enter over the inband stage with hardirqs off. */
+	if (irq_pipeline_debug()) {
+		WARN_ON_ONCE(!hard_irqs_disabled());
+		WARN_ON_ONCE(current_irq_stage != &inband_stage);
+	}
+
+	stage = top;
+
+	for (;;) {
+		if (stage == &inband_stage) {
+			if (test_inband_stall())
+				break;
+		} else {
+			if (test_oob_stall())
+				break;
+		}
+
+		p = this_staged(stage);
+		if (stage_irqs_pending(p)) {
+			if (stage == &inband_stage)
+				sync_current_irq_stage();
+			else {
+				/* Switch to oob before synchronizing. */
+				switch_oob(p);
+				sync_current_irq_stage();
+				/* Then back to the inband stage. */
+				switch_inband(this_inband_staged());
+			}
+		}
+
+		if (stage == &inband_stage)
+			break;
+
+		stage = &inband_stage;
+	}
+}
+
+void synchronize_pipeline(void) /* hardirqs off */
+{
+	struct irq_stage *top = &oob_stage;
+	int stalled = test_oob_stall();
+
+	if (unlikely(!oob_stage_present())) {
+		top = &inband_stage;
+		stalled = test_inband_stall();
+	}
+
+	if (current_irq_stage != top)
+		sync_irq_stage(top);
+	else if (!stalled)
+		sync_current_irq_stage();
+}
+
+static void __inband_irq_enable(void)
+{
+	struct irq_stage_data *p;
+	unsigned long flags;
+
+	check_inband_stage();
+
+	flags = hard_local_irq_save();
+
+	unstall_inband_nocheck();
+
+	p = this_inband_staged();
+	if (unlikely(stage_irqs_pending(p) && !in_pipeline())) {
+		sync_current_irq_stage();
+		hard_local_irq_restore(flags);
+		preempt_check_resched();
+	} else {
+		hard_local_irq_restore(flags);
+	}
+}
+
+/**
+ *	inband_irq_enable - enable interrupts for the inband stage
+ *
+ *	Enable interrupts for the inband stage, allowing interrupts to
+ *	preempt the in-band code. If in-band IRQs are pending for the
+ *	inband stage in the per-CPU log at the time of this call, they
+ *	are played back.
+ *
+ *      The caller is expected to tell the tracer about the change, by
+ *      calling trace_hardirqs_on().
+ */
+notrace void inband_irq_enable(void)
+{
+	/*
+	 * We are NOT supposed to enter this code with hard IRQs off.
+	 * If we do, then the caller might be wrongly assuming that
+	 * invoking local_irq_enable() implies enabling hard
+	 * interrupts like the legacy I-pipe did, which is not the
+	 * case anymore. Relax this requirement when oopsing, since
+	 * the kernel may be in a weird state.
+	 */
+	WARN_ON_ONCE(irq_pipeline_debug() && hard_irqs_disabled());
+	__inband_irq_enable();
+}
+EXPORT_SYMBOL(inband_irq_enable);
+
+/**
+ *	inband_irq_disable - disable interrupts for the inband stage
+ *
+ *	Disable interrupts for the inband stage, disabling in-band
+ *	interrupts. Out-of-band interrupts can still be taken and
+ *	delivered to their respective handlers though.
+ */
+notrace void inband_irq_disable(void)
+{
+	check_inband_stage();
+	stall_inband_nocheck();
+}
+EXPORT_SYMBOL(inband_irq_disable);
+
+/**
+ *	inband_irqs_disabled - test the virtual interrupt state
+ *
+ *	Returns non-zero if interrupts are currently disabled for the
+ *	inband stage, zero otherwise.
+ *
+ *	May be used from the oob stage too (e.g. for tracing
+ *	purpose).
+ */
+noinstr int inband_irqs_disabled(void)
+{
+	return test_inband_stall();
+}
+EXPORT_SYMBOL(inband_irqs_disabled);
+
+/**
+ *	inband_irq_save - test and disable (virtual) interrupts
+ *
+ *	Save the virtual interrupt state then disables interrupts for
+ *	the inband stage.
+ *
+ *      Returns the original interrupt state.
+ */
+trace_on_debug unsigned long inband_irq_save(void)
+{
+	check_inband_stage();
+	return test_and_stall_inband_nocheck();
+}
+EXPORT_SYMBOL(inband_irq_save);
+
+/**
+ *	inband_irq_restore - restore the (virtual) interrupt state
+ *      @x:	Interrupt state to restore
+ *
+ *	Restore the virtual interrupt state from x. If the inband
+ *	stage is unstalled as a consequence of this operation, any
+ *	interrupt pending for the inband stage in the per-CPU log is
+ *	played back.
+ */
+trace_on_debug void inband_irq_restore(unsigned long flags)
+{
+	if (flags)
+		inband_irq_disable();
+	else
+		__inband_irq_enable();
+}
+EXPORT_SYMBOL(inband_irq_restore);
+
+/**
+ *	oob_irq_enable - enable interrupts in the CPU
+ *
+ *	Enable interrupts in the CPU, allowing out-of-band interrupts
+ *	to preempt any code. If out-of-band IRQs are pending in the
+ *	per-CPU log for the oob stage at the time of this call, they
+ *	are played back.
+ */
+trace_on_debug void oob_irq_enable(void)
+{
+	struct irq_stage_data *p;
+
+	hard_local_irq_disable();
+
+	unstall_oob();
+
+	p = this_oob_staged();
+	if (unlikely(stage_irqs_pending(p)))
+		synchronize_pipeline();
+
+	hard_local_irq_enable();
+}
+EXPORT_SYMBOL(oob_irq_enable);
+
+/**
+ *	oob_irq_restore - restore the hardware interrupt state
+ *      @x:	Interrupt state to restore
+ *
+ *	Restore the harware interrupt state from x. If the oob stage
+ *	is unstalled as a consequence of this operation, any interrupt
+ *	pending for the oob stage in the per-CPU log is played back
+ *	prior to turning IRQs on.
+ *
+ *      NOTE: Stalling the oob stage must always be paired with
+ *      disabling hard irqs and conversely when calling
+ *      oob_irq_restore(), otherwise the latter would badly misbehave
+ *      in unbalanced conditions.
+ */
+trace_on_debug void __oob_irq_restore(unsigned long flags) /* hw interrupt off */
+{
+	struct irq_stage_data *p = this_oob_staged();
+
+	check_hard_irqs_disabled();
+
+	if (!flags) {
+		unstall_oob();
+		if (unlikely(stage_irqs_pending(p)))
+			synchronize_pipeline();
+		hard_local_irq_enable();
+	}
+}
+EXPORT_SYMBOL(__oob_irq_restore);
+
+/**
+ *	stage_disabled - test the interrupt state of the current stage
+ *
+ *	Returns non-zero if interrupts are currently disabled for the
+ *	current interrupt stage, zero otherwise.
+ *      In other words, returns non-zero either if:
+ *      - interrupts are disabled for the OOB context (i.e. hard disabled),
+ *      - the inband stage is current and inband interrupts are disabled.
+ */
+noinstr bool stage_disabled(void)
+{
+	bool ret = true;
+
+	if (!hard_irqs_disabled()) {
+		ret = false;
+		if (running_inband())
+			ret = test_inband_stall();
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(stage_disabled);
+
+/**
+ *	test_and_lock_stage - test and disable interrupts for the current stage
+ *	@irqsoff:	Pointer to boolean denoting stage_disabled()
+ *                      on entry
+ *
+ *	Fully disables interrupts for the current stage. When the
+ *	inband stage is current, the stall bit is raised and hardware
+ *	IRQs are masked as well. Only the latter operation is
+ *	performed when the oob stage is current.
+ *
+ *      Returns the combined interrupt state on entry including the
+ *      real/hardware (in CPU) and virtual (inband stage) states. For
+ *      this reason, [test_and_]lock_stage() must be paired with
+ *      unlock_stage() exclusively. The combined irq state returned by
+ *      the former may NOT be passed to hard_local_irq_restore().
+ *
+ *      The interrupt state of the current stage in the return value
+ *      (i.e. stall bit for the inband stage, hardware interrupt bit
+ *      for the oob stage) must be testable using
+ *      arch_irqs_disabled_flags().
+ *
+ *	Notice that test_and_lock_stage(), unlock_stage() are raw
+ *	level ops, which substitute to raw_local_irq_save(),
+ *	raw_local_irq_restore() in lockdep code. Therefore, changes to
+ *	the in-band stall bit must not be propagated to the tracing
+ *	core (i.e. no trace_hardirqs_*() annotations).
+ */
+noinstr unsigned long test_and_lock_stage(int *irqsoff)
+{
+	unsigned long flags;
+	int stalled, dummy;
+
+	if (irqsoff == NULL)
+		irqsoff = &dummy;
+
+	/*
+	 * Combine the hard irq flag and the stall bit into a single
+	 * state word. We need to fill in the stall bit only if the
+	 * inband stage is current, otherwise it is not relevant.
+	 */
+	flags = hard_local_irq_save();
+	*irqsoff = hard_irqs_disabled_flags(flags);
+	if (running_inband()) {
+		stalled = test_and_stall_inband_nocheck();
+		flags = irqs_merge_flags(flags, stalled);
+		if (stalled)
+			*irqsoff = 1;
+	}
+
+	/*
+	 * CAUTION: don't ever pass this verbatim to
+	 * hard_local_irq_restore(). Only unlock_stage() knows how to
+	 * decode and use a combined state word.
+	 */
+	return flags;
+}
+EXPORT_SYMBOL_GPL(test_and_lock_stage);
+
+/**
+ *	unlock_stage - restore interrupts for the current stage
+ *	@flags: 	Combined interrupt state to restore as received from
+ *              	test_and_lock_stage()
+ *
+ *	Restore the virtual interrupt state if the inband stage is
+ *      current, and the hardware interrupt state unconditionally.
+ *      The per-CPU log is not played for any stage.
+ */
+noinstr void unlock_stage(unsigned long irqstate)
+{
+	unsigned long flags = irqstate;
+	int stalled;
+
+	WARN_ON_ONCE(irq_pipeline_debug_locking() && !hard_irqs_disabled());
+
+	if (running_inband()) {
+		flags = irqs_split_flags(irqstate, &stalled);
+		if (!stalled)
+			unstall_inband_nocheck();
+	}
+
+	/*
+	 * The hardware interrupt bit is the only flag which may be
+	 * present in the combined state at this point, all other
+	 * status bits have been cleared by irqs_merge_flags(), so
+	 * don't ever try to reload the hardware status register with
+	 * such value directly!
+	 */
+	if (!hard_irqs_disabled_flags(flags))
+		hard_local_irq_enable();
+}
+EXPORT_SYMBOL_GPL(unlock_stage);
+
+/**
+ * sync_inband_irqs	- Synchronize the inband log
+ *
+ * Play any deferred interrupt which might have been logged for the
+ * in-band stage while running with hard irqs on but stalled.
+ *
+ * Called from the unstalled in-band stage. Returns with hard irqs off.
+ */
+void sync_inband_irqs(void)
+{
+	struct irq_stage_data *p;
+
+	check_inband_stage();
+	WARN_ON_ONCE(irq_pipeline_debug() && irqs_disabled());
+
+	if (!hard_irqs_disabled())
+		hard_local_irq_disable();
+
+	p = this_inband_staged();
+	if (unlikely(stage_irqs_pending(p))) {
+		/* Do not pile up preemption frames. */
+		preempt_disable_notrace();
+		sync_current_irq_stage();
+		preempt_enable_no_resched_notrace();
+	}
+}
+
+static inline bool irq_post_check(struct irq_stage *stage, unsigned int irq)
+{
+	if (irq_pipeline_debug()) {
+		if (WARN_ONCE(!hard_irqs_disabled(),
+				"hard irqs on posting IRQ%u to %s\n",
+				irq, stage->name))
+			return true;
+		if (WARN_ONCE(irq >= IRQ_BITMAP_BITS,
+				"cannot post invalid IRQ%u to %s\n",
+				irq, stage->name))
+			return true;
+	}
+
+	return false;
+}
+
+#if __IRQ_STAGE_MAP_LEVELS == 4
+
+/* Must be called hard irqs off. */
+void irq_post_stage(struct irq_stage *stage, unsigned int irq)
+{
+	struct irq_stage_data *p = this_staged(stage);
+	int l0b, l1b, l2b;
+
+	if (irq_post_check(stage, irq))
+		return;
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l2b = irq / BITS_PER_LONG;
+
+	__set_bit(irq, p->log.map->flat);
+	__set_bit(l2b, p->log.map->index_2);
+	__set_bit(l1b, p->log.map->index_1);
+	__set_bit(l0b, &p->log.index_0);
+}
+EXPORT_SYMBOL_GPL(irq_post_stage);
+
+#define ltob_1(__n)  ((__n) * BITS_PER_LONG)
+#define ltob_2(__n)  (ltob_1(__n) * BITS_PER_LONG)
+#define ltob_3(__n)  (ltob_2(__n) * BITS_PER_LONG)
+
+static inline int pull_next_irq(struct irq_stage_data *p)
+{
+	unsigned long l0m, l1m, l2m, l3m;
+	int l0b, l1b, l2b, l3b;
+	unsigned int irq;
+
+	l0m = p->log.index_0;
+	if (l0m == 0)
+		return -1;
+	l0b = __ffs(l0m);
+	irq = ltob_3(l0b);
+
+	l1m = p->log.map->index_1[l0b];
+	if (unlikely(l1m == 0)) {
+		WARN_ON_ONCE(1);
+		return -1;
+	}
+	l1b = __ffs(l1m);
+	irq += ltob_2(l1b);
+
+	l2m = p->log.map->index_2[ltob_1(l0b) + l1b];
+	if (unlikely(l2m == 0)) {
+		WARN_ON_ONCE(1);
+		return -1;
+	}
+	l2b = __ffs(l2m);
+	irq += ltob_1(l2b);
+
+	l3m = p->log.map->flat[ltob_2(l0b) + ltob_1(l1b) + l2b];
+	if (unlikely(l3m == 0))
+		return -1;
+	l3b = __ffs(l3m);
+	irq += l3b;
+
+	__clear_bit(irq, p->log.map->flat);
+	if (p->log.map->flat[irq / BITS_PER_LONG] == 0) {
+		__clear_bit(l2b, &p->log.map->index_2[ltob_1(l0b) + l1b]);
+		if (p->log.map->index_2[ltob_1(l0b) + l1b] == 0) {
+			__clear_bit(l1b, &p->log.map->index_1[l0b]);
+			if (p->log.map->index_1[l0b] == 0)
+				__clear_bit(l0b, &p->log.index_0);
+		}
+	}
+
+	return irq;
+}
+
+#elif __IRQ_STAGE_MAP_LEVELS == 3
+
+/* Must be called hard irqs off. */
+void irq_post_stage(struct irq_stage *stage, unsigned int irq)
+{
+	struct irq_stage_data *p = this_staged(stage);
+	int l0b, l1b;
+
+	if (irq_post_check(stage, irq))
+		return;
+
+	l0b = irq / (BITS_PER_LONG * BITS_PER_LONG);
+	l1b = irq / BITS_PER_LONG;
+
+	__set_bit(irq, p->log.map->flat);
+	__set_bit(l1b, p->log.map->index_1);
+	__set_bit(l0b, &p->log.index_0);
+}
+EXPORT_SYMBOL_GPL(irq_post_stage);
+
+static inline int pull_next_irq(struct irq_stage_data *p)
+{
+	unsigned long l0m, l1m, l2m;
+	int l0b, l1b, l2b, irq;
+
+	l0m = p->log.index_0;
+	if (unlikely(l0m == 0))
+		return -1;
+
+	l0b = __ffs(l0m);
+	l1m = p->log.map->index_1[l0b];
+	if (l1m == 0)
+		return -1;
+
+	l1b = __ffs(l1m) + l0b * BITS_PER_LONG;
+	l2m = p->log.map->flat[l1b];
+	if (unlikely(l2m == 0)) {
+		WARN_ON_ONCE(1);
+		return -1;
+	}
+
+	l2b = __ffs(l2m);
+	irq = l1b * BITS_PER_LONG + l2b;
+
+	__clear_bit(irq, p->log.map->flat);
+	if (p->log.map->flat[l1b] == 0) {
+		__clear_bit(l1b, p->log.map->index_1);
+		if (p->log.map->index_1[l0b] == 0)
+			__clear_bit(l0b, &p->log.index_0);
+	}
+
+	return irq;
+}
+
+#else /* __IRQ_STAGE_MAP_LEVELS == 2 */
+
+/* Must be called hard irqs off. */
+void irq_post_stage(struct irq_stage *stage, unsigned int irq)
+{
+	struct irq_stage_data *p = this_staged(stage);
+	int l0b = irq / BITS_PER_LONG;
+
+	if (irq_post_check(stage, irq))
+		return;
+
+	__set_bit(irq, p->log.map->flat);
+	__set_bit(l0b, &p->log.index_0);
+}
+EXPORT_SYMBOL_GPL(irq_post_stage);
+
+static inline int pull_next_irq(struct irq_stage_data *p)
+{
+	unsigned long l0m, l1m;
+	int l0b, l1b;
+
+	l0m = p->log.index_0;
+	if (l0m == 0)
+		return -1;
+
+	l0b = __ffs(l0m);
+	l1m = p->log.map->flat[l0b];
+	if (unlikely(l1m == 0)) {
+		WARN_ON_ONCE(1);
+		return -1;
+	}
+
+	l1b = __ffs(l1m);
+	__clear_bit(l1b, &p->log.map->flat[l0b]);
+	if (p->log.map->flat[l0b] == 0)
+		__clear_bit(l0b, &p->log.index_0);
+
+	return l0b * BITS_PER_LONG + l1b;
+}
+
+#endif  /* __IRQ_STAGE_MAP_LEVELS == 2 */
+
+/**
+ *	hard_preempt_disable - Disable preemption the hard way
+ *
+ *      Disable hardware interrupts in the CPU, and disable preemption
+ *      if currently running in-band code on the inband stage.
+ *
+ *      Return the hardware interrupt state.
+ */
+unsigned long hard_preempt_disable(void)
+{
+	unsigned long flags = hard_local_irq_save();
+
+	if (running_inband())
+		preempt_disable();
+
+	return flags;
+}
+EXPORT_SYMBOL_GPL(hard_preempt_disable);
+
+/**
+ *	hard_preempt_enable - Enable preemption the hard way
+ *
+ *      Enable preemption if currently running in-band code on the
+ *      inband stage, restoring the hardware interrupt state in the CPU.
+ *      The per-CPU log is not played for the oob stage.
+ */
+void hard_preempt_enable(unsigned long flags)
+{
+	if (running_inband()) {
+		preempt_enable_no_resched();
+		hard_local_irq_restore(flags);
+		if (!hard_irqs_disabled_flags(flags))
+			preempt_check_resched();
+	} else
+		hard_local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(hard_preempt_enable);
+
+static void handle_unexpected_irq(struct irq_desc *desc, irqreturn_t ret)
+{
+	unsigned int irq = irq_desc_get_irq(desc);
+	struct irqaction *action;
+
+	/*
+	 * Since IRQ_HANDLED was not received from any handler, we may
+	 * have a problem dealing with an OOB interrupt. The error
+	 * detection logic is as follows:
+	 *
+	 * - check and complain about any bogus return value from a
+	 * out-of-band IRQ handler: we only allow IRQ_HANDLED and
+	 * IRQ_NONE from those routines.
+	 *
+	 * - filter out spurious IRQs which may have been due to bus
+	 * asynchronicity, those tend to happen infrequently and
+	 * should not cause us to pull the break (see
+	 * note_interrupt()).
+	 *
+	 * - otherwise, stop pipelining the IRQ line after a thousand
+	 * consecutive unhandled events.
+	 *
+	 * NOTE: we should already be holding desc->lock for non
+	 * per-cpu IRQs, since we should only get there from the
+	 * pipeline entry context.
+	 */
+
+	WARN_ON_ONCE(irq_pipeline_debug() &&
+		     !irq_settings_is_per_cpu(desc) &&
+		     !raw_spin_is_locked(&desc->lock));
+
+	if (ret != IRQ_NONE) {
+		printk(KERN_ERR "out-of-band irq event %d: bogus return value %x\n",
+		       irq, ret);
+		for_each_action_of_desc(desc, action)
+			printk(KERN_ERR "[<%p>] %pf",
+			       action->handler, action->handler);
+		printk(KERN_CONT "\n");
+		return;
+	}
+
+	if (time_after(jiffies, desc->last_unhandled + HZ/10))
+		desc->irqs_unhandled = 0;
+	else
+		desc->irqs_unhandled++;
+
+	desc->last_unhandled = jiffies;
+
+	if (unlikely(desc->irqs_unhandled > 1000)) {
+		printk(KERN_ERR "out-of-band irq %d: stuck or unexpected\n", irq);
+		irq_settings_clr_oob(desc);
+		desc->istate |= IRQS_SPURIOUS_DISABLED;
+		irq_disable(desc);
+	}
+}
+
+static inline void incr_irq_kstat(struct irq_desc *desc)
+{
+	if (irq_settings_is_per_cpu_devid(desc))
+		__kstat_incr_irqs_this_cpu(desc);
+	else
+		kstat_incr_irqs_this_cpu(desc);
+}
+
+/*
+ * do_oob_irq() - Handles interrupts over the oob stage. Hard irqs
+ * off.
+ */
+static void do_oob_irq(struct irq_desc *desc)
+{
+	bool percpu_devid = irq_settings_is_per_cpu_devid(desc);
+	unsigned int irq = irq_desc_get_irq(desc);
+	irqreturn_t ret = IRQ_NONE, res;
+	struct irqaction *action;
+	void *dev_id;
+
+	action = desc->action;
+	if (unlikely(action == NULL))
+		goto done;
+
+	if (percpu_devid) {
+		trace_irq_handler_entry(irq, action);
+		dev_id = raw_cpu_ptr(action->percpu_dev_id);
+		ret = action->handler(irq, dev_id);
+		trace_irq_handler_exit(irq, action, ret);
+	} else {
+		desc->istate &= ~IRQS_PENDING;
+		if (unlikely(irqd_irq_disabled(&desc->irq_data)))
+			return;
+		irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
+		raw_spin_unlock(&desc->lock);
+		for_each_action_of_desc(desc, action) {
+			trace_irq_handler_entry(irq, action);
+			dev_id = action->dev_id;
+			res = action->handler(irq, dev_id);
+			trace_irq_handler_exit(irq, action, res);
+			ret |= res;
+		}
+		raw_spin_lock(&desc->lock);
+		irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
+	}
+done:
+	incr_irq_kstat(desc);
+
+	if (likely(ret & IRQ_HANDLED)) {
+		desc->irqs_unhandled = 0;
+		return;
+	}
+
+	handle_unexpected_irq(desc, ret);
+}
+
+/*
+ * Over the inband stage, IRQs must be dispatched by the arch-specific
+ * arch_do_IRQ_pipelined() routine.
+ *
+ * Entered with hardirqs on, inband stalled.
+ */
+static inline
+void do_inband_irq(struct irq_desc *desc)
+{
+	arch_do_IRQ_pipelined(desc);
+	WARN_ON_ONCE(irq_pipeline_debug() && !irqs_disabled());
+}
+
+static inline bool is_active_edge_event(struct irq_desc *desc)
+{
+	return (desc->istate & IRQS_PENDING) &&
+		!irqd_irq_disabled(&desc->irq_data);
+}
+
+bool handle_oob_irq(struct irq_desc *desc) /* hardirqs off */
+{
+	struct irq_stage_data *oobd = this_oob_staged();
+	unsigned int irq = irq_desc_get_irq(desc);
+	int stalled;
+
+	/*
+	 * Flow handlers of chained interrupts have no business
+	 * running here: they should decode the event, invoking
+	 * generic_handle_irq() for each cascaded IRQ.
+	 */
+	if (WARN_ON_ONCE(irq_pipeline_debug() &&
+			 irq_settings_is_chained(desc)))
+		return false;
+
+	/*
+	 * If no oob stage is present, all interrupts must go to the
+	 * inband stage through the interrupt log. Otherwise,
+	 * out-of-band IRQs are immediately delivered to the oob
+	 * stage, while in-band IRQs still go through the inband stage
+	 * log.
+	 *
+	 * This routine returns a boolean status telling the caller
+	 * whether an out-of-band interrupt was delivered.
+	 */
+	if (!oob_stage_present() || !irq_settings_is_oob(desc)) {
+		irq_post_stage(&inband_stage, irq);
+		return false;
+	}
+
+	if (WARN_ON_ONCE(irq_pipeline_debug() && running_inband()))
+		return false;
+
+	stalled = test_and_stall_oob();
+
+	if (unlikely(desc->istate & IRQS_EDGE)) {
+		do {
+			if (is_active_edge_event(desc))  {
+				if (irqd_irq_masked(&desc->irq_data))
+					unmask_irq(desc);
+			}
+			do_oob_irq(desc);
+		} while (is_active_edge_event(desc));
+	} else {
+		do_oob_irq(desc);
+	}
+
+	/*
+	 * Cascaded interrupts enter handle_oob_irq() on the stalled
+	 * out-of-band stage during the parent invocation. Make sure
+	 * to restore the stall bit accordingly.
+	 */
+	if (likely(!stalled))
+		unstall_oob();
+
+	/*
+	 * CPU migration and/or stage switching over the handler are
+	 * NOT allowed. These should take place over
+	 * irq_exit_pipeline().
+	 */
+	if (irq_pipeline_debug()) {
+		/* No CPU migration allowed. */
+		WARN_ON_ONCE(this_oob_staged() != oobd);
+		/* No stage migration allowed. */
+		WARN_ON_ONCE(current_irq_staged != oobd);
+	}
+
+	return true;
+}
+
+static inline
+void copy_timer_regs(struct irq_desc *desc, struct pt_regs *regs)
+{
+	struct irq_pipeline_data *p;
+
+	if (desc->action == NULL || !(desc->action->flags & __IRQF_TIMER))
+		return;
+	/*
+	 * Given our deferred dispatching model for regular IRQs, we
+	 * record the preempted context registers only for the latest
+	 * timer interrupt, so that the regular tick handler charges
+	 * CPU times properly. It is assumed that no other interrupt
+	 * handler cares for such information.
+	 */
+	p = raw_cpu_ptr(&irq_pipeline);
+	arch_save_timer_regs(&p->tick_regs, regs);
+}
+
+static __always_inline
+struct irq_stage_data *switch_stage_on_irq(void)
+{
+	struct irq_stage_data *prevd = current_irq_staged, *nextd;
+
+	if (oob_stage_present()) {
+		nextd = this_oob_staged();
+		if (prevd != nextd)
+			switch_oob(nextd);
+	}
+
+	return prevd;
+}
+
+static __always_inline
+void restore_stage_on_irq(struct irq_stage_data *prevd)
+{
+	/*
+	 * CPU migration and/or stage switching over
+	 * irq_exit_pipeline() are allowed.  Our exit logic is as
+	 * follows:
+	 *
+	 *    ENTRY      EXIT      EPILOGUE
+	 *
+	 *    oob        oob       nop
+	 *    inband     oob       switch inband
+	 *    oob        inband    nop
+	 *    inband     inband    nop
+	 */
+	if (prevd->stage == &inband_stage &&
+		current_irq_staged == this_oob_staged())
+		switch_inband(this_inband_staged());
+}
+
+/**
+ *	generic_pipeline_irq_desc - Pass an IRQ to the pipeline
+ *	@desc:	Descriptor of the IRQ to pass
+ *	@regs:	Register file coming from the low-level handling code
+ *
+ *	Inject an IRQ into the pipeline from a CPU interrupt or trap
+ *	context.  A flow handler runs next for this IRQ.
+ *
+ *      Hard irqs must be off on entry. Caller should have pushed the
+ *      IRQ regs using set_irq_regs().
+ */
+void generic_pipeline_irq_desc(struct irq_desc *desc, struct pt_regs *regs)
+{
+	int irq = irq_desc_get_irq(desc);
+
+	if (irq_pipeline_debug() && !hard_irqs_disabled()) {
+		hard_local_irq_disable();
+		pr_err("IRQ pipeline: interrupts enabled on entry (IRQ%u)\n", irq);
+	}
+
+	trace_irq_pipeline_entry(irq);
+	copy_timer_regs(desc, regs);
+	generic_handle_irq_desc(desc);
+	trace_irq_pipeline_exit(irq);
+}
+
+void generic_pipeline_irq(unsigned int irq, struct pt_regs *regs)
+{
+	struct irq_desc *desc = irq_to_cached_desc(irq);
+	struct pt_regs *old_regs;
+
+	old_regs = set_irq_regs(regs);
+	generic_pipeline_irq_desc(desc, regs);
+	set_irq_regs(old_regs);
+}
+
+struct irq_stage_data *handle_irq_pipelined_prepare(struct pt_regs *regs)
+{
+	struct irq_stage_data *prevd;
+
+	/*
+	 * Running with the oob stage stalled implies hardirqs off.
+	 * For this reason, if the oob stage is stalled when we
+	 * receive an interrupt from the hardware, something is badly
+	 * broken in our interrupt state. Try fixing up, but without
+	 * great hopes.
+	 */
+	if (irq_pipeline_debug()) {
+		if (test_oob_stall()) {
+			pr_err("IRQ pipeline: out-of-band stage stalled on IRQ entry\n");
+			unstall_oob();
+		}
+		WARN_ON(on_pipeline_entry());
+	}
+
+	/*
+	 * Switch early on to the out-of-band stage if present,
+	 * anticipating a companion kernel is going to handle the
+	 * incoming event. If not, never mind, we will switch back
+	 * in-band before synchronizing interrupts.
+	 */
+	prevd = switch_stage_on_irq();
+
+	/* Tell the companion core about the entry. */
+	irq_enter_pipeline();
+
+	/*
+	 * Invariant: IRQs may not pile up in the section covered by
+	 * the PIPELINE_OFFSET marker, because:
+	 *
+	 * - out-of-band handlers called from handle_oob_irq() may NOT
+	 * re-enable hard interrupts. Ever.
+	 *
+	 * - synchronizing the in-band log with hard interrupts
+	 * enabled is done outside of this section.
+	 */
+	preempt_count_add(PIPELINE_OFFSET);
+
+	/*
+	 * From the standpoint of the in-band context when pipelining
+	 * is in effect, an interrupt entry is unsafe in a similar way
+	 * a NMI is, since it may preempt almost anywhere as IRQs are
+	 * only virtually masked most of the time, including inside
+	 * (virtually) interrupt-free sections. Declare a NMI entry so
+	 * that the low handling code is allowed to enter RCU read
+	 * sides (e.g. handle_domain_irq() needs this to resolve IRQ
+	 * mappings).
+	 */
+	rcu_nmi_enter();
+
+	return prevd;
+}
+
+int handle_irq_pipelined_finish(struct irq_stage_data *prevd,
+				struct pt_regs *regs)
+{
+	/*
+	 * Leave the (pseudo-)NMI entry for RCU before the out-of-band
+	 * core might reschedule in irq_exit_pipeline(), and
+	 * interrupts are hard enabled again on this CPU as a result
+	 * of switching context.
+	 */
+	rcu_nmi_exit();
+
+	/*
+	 * Make sure to leave the pipeline entry context before
+	 * allowing the companion core to reschedule, and eventually
+	 * synchronizing interrupts.
+	 */
+	preempt_count_sub(PIPELINE_OFFSET);
+
+	/* Allow the companion core to reschedule. */
+	irq_exit_pipeline();
+
+	/* Back to the preempted stage. */
+	restore_stage_on_irq(prevd);
+
+	/*
+	 * We have to synchronize interrupts because some might have
+	 * been logged while we were busy handling an out-of-band
+	 * event coming from the hardware:
+	 *
+	 * - as a result of calling an out-of-band handler which in
+	 * turn posted them.
+	 *
+	 * - because we posted them directly for scheduling the
+	 * interrupt to happen from the in-band stage.
+	 */
+	synchronize_pipeline_on_irq();
+
+#ifdef CONFIG_DOVETAIL
+	/*
+	 * Sending MAYDAY is in essence a rare case, so prefer test
+	 * then maybe clear over test_and_clear.
+	 */
+	if (user_mode(regs) && test_thread_flag(TIF_MAYDAY))
+		dovetail_call_mayday(regs);
+#endif
+
+	return running_inband() && !irqs_disabled();
+}
+
+int handle_irq_pipelined(struct pt_regs *regs)
+{
+	struct irq_stage_data *prevd;
+
+	prevd = handle_irq_pipelined_prepare(regs);
+	handle_arch_irq(regs);
+	return handle_irq_pipelined_finish(prevd, regs);
+}
+
+/**
+ *	irq_inject_pipeline - Inject a software-generated IRQ into the
+ *	pipeline @irq: IRQ to inject
+ *
+ *	Inject an IRQ into the pipeline by software as if such
+ *	hardware event had happened on the current CPU.
+ */
+int irq_inject_pipeline(unsigned int irq)
+{
+	struct irq_stage_data *oobd, *prevd;
+	struct irq_desc *desc;
+	unsigned long flags;
+
+	desc = irq_to_cached_desc(irq);
+	if (desc == NULL)
+		return -EINVAL;
+
+	flags = hard_local_irq_save();
+
+	/*
+	 * Handle the case of an IRQ sent to a stalled oob stage here,
+	 * which allows to trap the same condition in handle_oob_irq()
+	 * in a debug check (see comment there).
+	 */
+	oobd = this_oob_staged();
+	if (oob_stage_present() &&
+		irq_settings_is_oob(desc) &&
+		test_oob_stall()) {
+		irq_post_stage(&oob_stage, irq);
+	} else {
+		prevd = switch_stage_on_irq();
+		irq_enter_pipeline();
+		handle_oob_irq(desc);
+		irq_exit_pipeline();
+		restore_stage_on_irq(prevd);
+		synchronize_pipeline_on_irq();
+	}
+
+	hard_local_irq_restore(flags);
+
+	return 0;
+
+}
+EXPORT_SYMBOL_GPL(irq_inject_pipeline);
+
+/*
+ * sync_current_irq_stage() -- Flush the pending IRQs for the current
+ * stage (and processor). This routine flushes the interrupt log (see
+ * "Optimistic interrupt protection" from D. Stodolsky et al. for more
+ * on the deferred interrupt scheme). Every interrupt which has
+ * occurred while the pipeline was stalled gets played.
+ *
+ * CAUTION: CPU migration may occur over this routine if running over
+ * the inband stage.
+ */
+void sync_current_irq_stage(void) /* hard irqs off */
+{
+	struct irq_stage_data *p;
+	struct irq_stage *stage;
+	struct irq_desc *desc;
+	int irq;
+
+	WARN_ON_ONCE(irq_pipeline_debug() && on_pipeline_entry());
+	check_hard_irqs_disabled();
+
+	p = current_irq_staged;
+respin:
+	stage = p->stage;
+	if (stage == &inband_stage) {
+		/*
+		 * Since we manipulate the stall bit directly, we have
+		 * to open code the IRQ state tracing.
+		 */
+		stall_inband_nocheck();
+		trace_hardirqs_off();
+	} else {
+		stall_oob();
+	}
+
+	for (;;) {
+		irq = pull_next_irq(p);
+		if (irq < 0)
+			break;
+		/*
+		 * Make sure the compiler does not reorder wrongly, so
+		 * that all updates to maps are done before the
+		 * handler gets called.
+		 */
+		barrier();
+
+		desc = irq_to_cached_desc(irq);
+
+		if (stage == &inband_stage) {
+			hard_local_irq_enable();
+			do_inband_irq(desc);
+			hard_local_irq_disable();
+		} else {
+			do_oob_irq(desc);
+		}
+
+		/*
+		 * We might have switched from the oob stage to the
+		 * in-band one on return from the handler, in which
+		 * case we might also have migrated to a different CPU
+		 * (the converse in-band -> oob switch is NOT allowed
+		 * though). Reload the current per-cpu context
+		 * pointer, so that we further pull pending interrupts
+		 * from the proper in-band log.
+		 */
+		p = current_irq_staged;
+		if (p->stage != stage) {
+			if (WARN_ON_ONCE(irq_pipeline_debug() &&
+					stage == &inband_stage))
+				break;
+			goto respin;
+		}
+	}
+
+	if (stage == &inband_stage) {
+		trace_hardirqs_on();
+		unstall_inband_nocheck();
+	} else {
+		unstall_oob();
+	}
+}
+
+#ifndef CONFIG_GENERIC_ENTRY
+
+/*
+ * These helpers are normally called from the kernel entry/exit code
+ * in the asm section by architectures which do not use the generic
+ * kernel entry code, in order to save the interrupt and lockdep
+ * states for the in-band stage on entry, restoring them when leaving
+ * the kernel.  The per-architecture arch_kentry_set/get_irqstate()
+ * calls determine where this information should be kept while running
+ * in kernel context, indexed on the current register frame.
+ */
+
+#define KENTRY_STALL_BIT      BIT(0) /* Tracks INBAND_STALL_BIT */
+#define KENTRY_LOCKDEP_BIT    BIT(1) /* Tracks hardirqs_enabled */
+
+asmlinkage __visible noinstr void kentry_enter_pipelined(struct pt_regs *regs)
+{
+	long irqstate = 0;
+
+	WARN_ON(irq_pipeline_debug() && !hard_irqs_disabled());
+
+	if (!running_inband())
+		return;
+
+	if (lockdep_read_irqs_state())
+		irqstate |= KENTRY_LOCKDEP_BIT;
+
+	if (irqs_disabled())
+		irqstate |= KENTRY_STALL_BIT;
+	else
+		trace_hardirqs_off();
+
+	arch_kentry_set_irqstate(regs, irqstate);
+}
+
+asmlinkage void __visible noinstr kentry_exit_pipelined(struct pt_regs *regs)
+{
+	long irqstate;
+
+	WARN_ON(irq_pipeline_debug() && !hard_irqs_disabled());
+
+	if (!running_inband())
+		return;
+
+	/*
+	 * If the in-band stage of the kernel is current but the IRQ
+	 * is not going to be delivered because the latter is stalled,
+	 * keep the tracing logic unaware of the receipt, so that no
+	 * false positive is triggered in lockdep (e.g. IN-HARDIRQ-W
+	 * -> HARDIRQ-ON-W). In this case, we still have to restore
+	 * the lockdep irq state independently, since it might not be
+	 * in sync with the stall bit (e.g. raw_local_irq_disable/save
+	 * do flip the stall bit, but are not tracked by lockdep).
+	 */
+
+	irqstate = arch_kentry_get_irqstate(regs);
+	if (!(irqstate & KENTRY_STALL_BIT)) {
+		stall_inband_nocheck();
+		trace_hardirqs_on();
+		unstall_inband_nocheck();
+	} else {
+		lockdep_write_irqs_state(!!(irqstate & KENTRY_LOCKDEP_BIT));
+	}
+}
+
+#endif /* !CONFIG_GENERIC_ENTRY */
+
+/**
+ *      run_oob_call - escalate function call to the oob stage
+ *      @fn:    address of routine
+ *      @arg:   routine argument
+ *
+ *      Make the specified function run on the oob stage, switching
+ *      the current stage accordingly if needed. The escalated call is
+ *      allowed to perform a stage migration in the process.
+ */
+int notrace run_oob_call(int (*fn)(void *arg), void *arg)
+{
+	struct irq_stage_data *p, *old;
+	struct irq_stage *oob;
+	unsigned long flags;
+	int ret, s;
+
+	flags = hard_local_irq_save();
+
+	/* Switch to the oob stage if not current. */
+	p = this_oob_staged();
+	oob = p->stage;
+	old = current_irq_staged;
+	if (old != p)
+		switch_oob(p);
+
+	s = test_and_stall_oob();
+	barrier();
+	ret = fn(arg);
+	hard_local_irq_disable();
+	if (!s)
+		unstall_oob();
+
+	/*
+	 * The exit logic is as follows:
+	 *
+	 *    ON-ENTRY  AFTER-CALL  EPILOGUE
+	 *
+	 *    oob       oob         sync current stage if !stalled
+	 *    inband    oob         switch to inband + sync all stages
+	 *    oob       inband      sync all stages
+	 *    inband    inband      sync all stages
+	 *
+	 * Each path which has stalled the oob stage while running on
+	 * the inband stage at some point during the escalation
+	 * process must synchronize all stages of the pipeline on
+	 * exit. Otherwise, we may restrict the synchronization scope
+	 * to the current stage when the whole sequence ran on the oob
+	 * stage.
+	 */
+	p = this_oob_staged();
+	if (likely(current_irq_staged == p)) {
+		if (old->stage == oob) {
+			if (!s && stage_irqs_pending(p))
+				sync_current_irq_stage();
+			goto out;
+		}
+		switch_inband(this_inband_staged());
+	}
+
+	sync_irq_stage(oob);
+out:
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(run_oob_call);
+
+int enable_oob_stage(const char *name)
+{
+	struct irq_event_map *map;
+	struct irq_stage_data *p;
+	int cpu, ret;
+
+	if (oob_stage_present())
+		return -EBUSY;
+
+	/* Set up the out-of-band interrupt stage on all CPUs. */
+
+	for_each_possible_cpu(cpu) {
+		p = &per_cpu(irq_pipeline.stages, cpu)[1];
+		map = p->log.map; /* save/restore after memset(). */
+		memset(p, 0, sizeof(*p));
+		p->stage = &oob_stage;
+		memset(map, 0, sizeof(struct irq_event_map));
+		p->log.map = map;
+#ifdef CONFIG_DEBUG_IRQ_PIPELINE
+		p->cpu = cpu;
+#endif
+	}
+
+	ret = arch_enable_oob_stage();
+	if (ret)
+		return ret;
+
+	oob_stage.name = name;
+	smp_wmb();
+	oob_stage.index = 1;
+
+	pr_info("IRQ pipeline: high-priority %s stage added.\n", name);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(enable_oob_stage);
+
+void disable_oob_stage(void)
+{
+	const char *name = oob_stage.name;
+
+	WARN_ON(!running_inband() || !oob_stage_present());
+
+	oob_stage.index = 0;
+	smp_wmb();
+
+	pr_info("IRQ pipeline: %s stage removed.\n", name);
+}
+EXPORT_SYMBOL_GPL(disable_oob_stage);
+
+void irq_pipeline_oops(void)
+{
+	irq_pipeline_oopsing = true;
+	local_irq_disable_full();
+}
+
+/*
+ * Used to save/restore the status bits of the inband stage across runs
+ * of NMI-triggered code, so that we can restore the original pipeline
+ * state before leaving NMI context.
+ */
+static DEFINE_PER_CPU(unsigned long, nmi_saved_stall_bits);
+
+noinstr void irq_pipeline_nmi_enter(void)
+{
+	raw_cpu_write(nmi_saved_stall_bits, current->stall_bits);
+
+}
+EXPORT_SYMBOL(irq_pipeline_nmi_enter);
+
+noinstr void irq_pipeline_nmi_exit(void)
+{
+	current->stall_bits = raw_cpu_read(nmi_saved_stall_bits);
+}
+EXPORT_SYMBOL(irq_pipeline_nmi_exit);
+
+bool __weak irq_cpuidle_control(struct cpuidle_device *dev,
+				struct cpuidle_state *state)
+{
+	/*
+	 * Allow entering the idle state by default, matching the
+	 * original behavior when CPU_IDLE is turned
+	 * on. irq_cpuidle_control() may be overriden by an
+	 * out-of-band code for determining whether the CPU may
+	 * actually enter the idle state.
+	 */
+	return true;
+}
+
+/**
+ *	irq_cpuidle_enter - Prepare for entering the next idle state
+ *	@dev: CPUIDLE device
+ *	@state: CPUIDLE state to be entered
+ *
+ *	Flush the in-band interrupt log before the caller idles, so
+ *	that no event lingers before we actually wait for the next
+ *	IRQ, in which case we ask the caller to abort the idling
+ *	process altogether. The companion core is also given the
+ *	opportunity to block the idling process by having
+ *	irq_cpuidle_control() return @false.
+ *
+ *	Returns @true if caller may proceed with idling, @false
+ *	otherwise. The in-band log is guaranteed empty on return, hard
+ *	irqs left off so that no event might sneak in until the caller
+ *	actually idles.
+ */
+bool irq_cpuidle_enter(struct cpuidle_device *dev,
+		       struct cpuidle_state *state)
+{
+	WARN_ON_ONCE(irq_pipeline_debug() && !irqs_disabled());
+
+	hard_local_irq_disable();
+
+	if (stage_irqs_pending(this_inband_staged())) {
+		unstall_inband_nocheck();
+		synchronize_pipeline();
+		stall_inband_nocheck();
+		trace_hardirqs_off();
+		return false;
+	}
+
+	return irq_cpuidle_control(dev, state);
+}
+
+static unsigned int inband_work_sirq;
+
+static irqreturn_t inband_work_interrupt(int sirq, void *dev_id)
+{
+	irq_work_run();
+
+	return IRQ_HANDLED;
+}
+
+static struct irqaction inband_work = {
+	.handler = inband_work_interrupt,
+	.name = "in-band work",
+	.flags = IRQF_NO_THREAD,
+};
+
+void irq_local_work_raise(void)
+{
+	unsigned long flags;
+
+	/*
+	 * irq_work_queue() may be called from the in-band stage too
+	 * in case we want to delay a work until the hard irqs are on
+	 * again, so we may only sync the in-band log when unstalled,
+	 * with hard irqs on.
+	 */
+	flags = hard_local_irq_save();
+	irq_post_inband(inband_work_sirq);
+	if (running_inband() &&
+	    !hard_irqs_disabled_flags(flags) && !irqs_disabled())
+		sync_current_irq_stage();
+	hard_local_irq_restore(flags);
+}
+
+#ifdef CONFIG_DEBUG_IRQ_PIPELINE
+
+#ifdef CONFIG_LOCKDEP
+static inline bool lockdep_on_error(void)
+{
+	return !debug_locks;
+}
+#else
+static inline bool lockdep_on_error(void)
+{
+	return false;
+}
+#endif
+
+notrace void check_inband_stage(void)
+{
+	struct irq_stage *this_stage;
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+
+	this_stage = current_irq_stage;
+	if (likely(this_stage == &inband_stage && !test_oob_stall())) {
+		hard_local_irq_restore(flags);
+		return;
+	}
+
+	if (in_nmi() || irq_pipeline_oopsing || lockdep_on_error()) {
+		hard_local_irq_restore(flags);
+		return;
+	}
+
+	/*
+	 * This will disable all further pipeline debug checks, since
+	 * a wrecked interrupt state is likely to trigger many of
+	 * them, ending up in a terrible mess. IOW, the current
+	 * situation must be fixed prior to investigating any
+	 * subsequent issue that might still exist.
+	 */
+	irq_pipeline_oopsing = true;
+
+	hard_local_irq_restore(flags);
+
+	if (this_stage != &inband_stage)
+		pr_err("IRQ pipeline: some code running in oob context '%s'\n"
+		       "              called an in-band only routine\n",
+		       this_stage->name);
+	else
+		pr_err("IRQ pipeline: oob stage found stalled while modifying in-band\n"
+		       "              interrupt state and/or running sleeping code\n");
+
+	dump_stack();
+}
+EXPORT_SYMBOL(check_inband_stage);
+
+void check_spinlock_context(void)
+{
+	WARN_ON_ONCE(in_pipeline() || running_oob());
+
+}
+EXPORT_SYMBOL(check_spinlock_context);
+
+#endif /* CONFIG_DEBUG_IRQ_PIPELINE */
+
+static inline void fixup_percpu_data(void)
+{
+#ifdef CONFIG_SMP
+	struct irq_pipeline_data *p;
+	int cpu;
+
+	/*
+	 * A temporary event log is used by the inband stage during the
+	 * early boot up (bootup_irq_map), until the per-cpu areas
+	 * have been set up.
+	 *
+	 * Obviously, this code must run over the boot CPU, before SMP
+	 * operations start, with hard IRQs off so that nothing can
+	 * change under our feet.
+	 */
+	WARN_ON(!hard_irqs_disabled());
+
+	memcpy(&per_cpu(irq_map_array, 0)[0], &bootup_irq_map,
+	       sizeof(struct irq_event_map));
+
+	for_each_possible_cpu(cpu) {
+		p = &per_cpu(irq_pipeline, cpu);
+		p->stages[0].stage = &inband_stage;
+		p->stages[0].log.map = &per_cpu(irq_map_array, cpu)[0];
+		p->stages[1].log.map = &per_cpu(irq_map_array, cpu)[1];
+#ifdef CONFIG_DEBUG_IRQ_PIPELINE
+		p->stages[0].cpu = cpu;
+		p->stages[1].cpu = cpu;
+#endif
+	}
+#endif
+}
+
+void __init irq_pipeline_init_early(void)
+{
+	/*
+	 * This is called early from start_kernel(), even before the
+	 * actual number of IRQs is known. We are running on the boot
+	 * CPU, hw interrupts are off, and secondary CPUs are still
+	 * lost in space. Careful.
+	 */
+	fixup_percpu_data();
+}
+
+/**
+ *	irq_pipeline_init - Main pipeline core inits
+ *
+ *	This is step #2 of the 3-step pipeline initialization, which
+ *	should happen right after init_IRQ() has run. The internal
+ *	service interrupts are created along with the synthetic IRQ
+ *	domain, and the arch-specific init chores are performed too.
+ *
+ *	Interrupt pipelining should be fully functional when this
+ *	routine returns.
+ */
+void __init irq_pipeline_init(void)
+{
+	WARN_ON(!hard_irqs_disabled());
+
+	synthetic_irq_domain = irq_domain_add_nomap(NULL, ~0,
+						    &sirq_domain_ops,
+						    NULL);
+	inband_work_sirq = irq_create_direct_mapping(synthetic_irq_domain);
+	setup_percpu_irq(inband_work_sirq, &inband_work);
+
+	/*
+	 * We are running on the boot CPU, hw interrupts are off, and
+	 * secondary CPUs are still lost in space. Now we may run
+	 * arch-specific code for enabling the pipeline.
+	 */
+	arch_irq_pipeline_init();
+
+	irq_pipeline_active = true;
+
+	pr_info("IRQ pipeline enabled\n");
+}
+
+#ifndef CONFIG_SPARSE_IRQ
+EXPORT_SYMBOL_GPL(irq_desc);
+#endif
diff --git a/kernel/kernel/irq/proc.c b/kernel/kernel/irq/proc.c
index 0459b69835..cad725f0bc 100644
--- a/kernel/kernel/irq/proc.c
+++ b/kernel/kernel/irq/proc.c
@@ -517,6 +517,9 @@ int show_interrupts(struct seq_file *p, void *v)
 		seq_printf(p, " %*s", prec, "");
 #ifdef CONFIG_GENERIC_IRQ_SHOW_LEVEL
 	seq_printf(p, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge");
+#endif
+#ifdef CONFIG_IRQ_PIPELINE
+	seq_printf(p, " %-3s", irq_settings_is_oob(desc) ? "oob" : "");
 #endif
 	if (desc->name)
 		seq_printf(p, "-%-8s", desc->name);
diff --git a/kernel/kernel/irq/resend.c b/kernel/kernel/irq/resend.c
index 8ccd32a0cc..01b9f2383c 100644
--- a/kernel/kernel/irq/resend.c
+++ b/kernel/kernel/irq/resend.c
@@ -16,10 +16,11 @@
 #include <linux/module.h>
 #include <linux/random.h>
 #include <linux/interrupt.h>
+#include <linux/irq_pipeline.h>
 
 #include "internals.h"
 
-#ifdef CONFIG_HARDIRQS_SW_RESEND
+#if defined(CONFIG_HARDIRQS_SW_RESEND) && !defined(CONFIG_IRQ_PIPELINE)
 
 /* Bitmap to handle software resend of interrupts: */
 static DECLARE_BITMAP(irqs_resend, IRQ_BITMAP_BITS);
@@ -82,7 +83,12 @@ static int irq_sw_resend(struct irq_desc *desc)
 #else
 static int irq_sw_resend(struct irq_desc *desc)
 {
+#if defined(CONFIG_HARDIRQS_SW_RESEND) && defined(CONFIG_IRQ_PIPELINE)
+	irq_inject_pipeline(irq_desc_get_irq(desc));
+	return 0;
+#else
 	return -EINVAL;
+#endif
 }
 #endif
 
diff --git a/kernel/kernel/irq/settings.h b/kernel/kernel/irq/settings.h
index 0033d459fd..aa9755600e 100644
--- a/kernel/kernel/irq/settings.h
+++ b/kernel/kernel/irq/settings.h
@@ -19,6 +19,8 @@ enum {
 	_IRQ_DISABLE_UNLAZY	= IRQ_DISABLE_UNLAZY,
 	_IRQ_HIDDEN		= IRQ_HIDDEN,
 	_IRQ_RAW		= IRQ_RAW,
+	_IRQ_OOB		= IRQ_OOB,
+	_IRQ_CHAINED		= IRQ_CHAINED,
 	_IRQF_MODIFY_MASK	= IRQF_MODIFY_MASK,
 };
 
@@ -35,6 +37,8 @@ enum {
 #define IRQ_DISABLE_UNLAZY	GOT_YOU_MORON
 #define IRQ_HIDDEN		GOT_YOU_MORON
 #define IRQ_RAW			GOT_YOU_MORON
+#define IRQ_OOB			GOT_YOU_MORON
+#define IRQ_CHAINED		GOT_YOU_MORON
 #undef IRQF_MODIFY_MASK
 #define IRQF_MODIFY_MASK	GOT_YOU_MORON
 
@@ -195,3 +199,33 @@ static inline bool irq_settings_is_raw(struct irq_desc *desc)
 	WARN_ON_ONCE(1);
 	return false;
 }
+
+static inline bool irq_settings_is_oob(struct irq_desc *desc)
+{
+	return desc->status_use_accessors & _IRQ_OOB;
+}
+
+static inline void irq_settings_clr_oob(struct irq_desc *desc)
+{
+	desc->status_use_accessors &= ~_IRQ_OOB;
+}
+
+static inline void irq_settings_set_oob(struct irq_desc *desc)
+{
+	desc->status_use_accessors |= _IRQ_OOB;
+}
+
+static inline bool irq_settings_is_chained(struct irq_desc *desc)
+{
+	return desc->status_use_accessors & _IRQ_CHAINED;
+}
+
+static inline void irq_settings_set_chained(struct irq_desc *desc)
+{
+	desc->status_use_accessors |= _IRQ_CHAINED;
+}
+
+static inline void irq_settings_clr_chained(struct irq_desc *desc)
+{
+	desc->status_use_accessors &= ~_IRQ_CHAINED;
+}
diff --git a/kernel/kernel/irq_work.c b/kernel/kernel/irq_work.c
index e0ed16db66..19417ce46d 100644
--- a/kernel/kernel/irq_work.c
+++ b/kernel/kernel/irq_work.c
@@ -49,6 +49,11 @@ void __weak arch_irq_work_raise(void)
 	 */
 }
 
+void __weak irq_local_work_raise(void)
+{
+	arch_irq_work_raise();
+}
+
 /* Enqueue on current CPU, work must already be claimed and preempt disabled */
 static void __irq_work_queue_local(struct irq_work *work)
 {
@@ -56,10 +61,10 @@ static void __irq_work_queue_local(struct irq_work *work)
 	if (atomic_read(&work->flags) & IRQ_WORK_LAZY) {
 		if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
 		    tick_nohz_tick_stopped())
-			arch_irq_work_raise();
+			irq_local_work_raise();
 	} else {
 		if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
-			arch_irq_work_raise();
+			irq_local_work_raise();
 	}
 }
 
diff --git a/kernel/kernel/kthread.c b/kernel/kernel/kthread.c
index 9d736f57b8..896383ba98 100644
--- a/kernel/kernel/kthread.c
+++ b/kernel/kernel/kthread.c
@@ -14,6 +14,7 @@
 #include <linux/sched/mm.h>
 #include <linux/sched/task.h>
 #include <linux/kthread.h>
+#include <linux/irq_pipeline.h>
 #include <linux/completion.h>
 #include <linux/err.h>
 #include <linux/cgroup.h>
@@ -1331,6 +1332,7 @@ void kthread_use_mm(struct mm_struct *mm)
 {
 	struct mm_struct *active_mm;
 	struct task_struct *tsk = current;
+	unsigned long flags;
 
 	WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
 	WARN_ON_ONCE(tsk->mm);
@@ -1339,12 +1341,14 @@ void kthread_use_mm(struct mm_struct *mm)
 	/* Hold off tlb flush IPIs while switching mm's */
 	local_irq_disable();
 	active_mm = tsk->active_mm;
+	protect_inband_mm(flags);
 	if (active_mm != mm) {
 		mmgrab(mm);
 		tsk->active_mm = mm;
 	}
 	tsk->mm = mm;
 	switch_mm_irqs_off(active_mm, mm, tsk);
+	unprotect_inband_mm(flags);
 	local_irq_enable();
 	task_unlock(tsk);
 #ifdef finish_arch_post_lock_switch
diff --git a/kernel/kernel/locking/Makefile b/kernel/kernel/locking/Makefile
index 6d11cfb9b4..c491040bde 100644
--- a/kernel/kernel/locking/Makefile
+++ b/kernel/kernel/locking/Makefile
@@ -28,6 +28,7 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
 obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
 obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
 obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o
+obj-$(CONFIG_IRQ_PIPELINE) += pipeline.o
 obj-$(CONFIG_QUEUED_RWLOCKS) += qrwlock.o
 obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o
 obj-$(CONFIG_WW_MUTEX_SELFTEST) += test-ww_mutex.o
diff --git a/kernel/kernel/locking/lockdep.c b/kernel/kernel/locking/lockdep.c
index 6cbd2b4444..48f5a6bb55 100644
--- a/kernel/kernel/locking/lockdep.c
+++ b/kernel/kernel/locking/lockdep.c
@@ -42,6 +42,7 @@
 #include <linux/stacktrace.h>
 #include <linux/debug_locks.h>
 #include <linux/irqflags.h>
+#include <linux/irqstage.h>
 #include <linux/utsname.h>
 #include <linux/hash.h>
 #include <linux/ftrace.h>
@@ -104,9 +105,56 @@ static __always_inline bool lockdep_enabled(void)
 static arch_spinlock_t __lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
 static struct task_struct *__owner;
 
+static __always_inline bool lockdep_stage_disabled(void)
+{
+	return stage_disabled();
+}
+
+#ifdef CONFIG_IRQ_PIPELINE
+/*
+ * If LOCKDEP is enabled, we want irqs to be disabled for both stages
+ * when traversing the lockdep code for hard and mutable locks (at the
+ * expense of massive latency overhead though).
+ */
+static __always_inline unsigned long lockdep_stage_test_and_disable(int *irqsoff)
+{
+	return test_and_lock_stage(irqsoff);
+}
+
+static __always_inline unsigned long lockdep_stage_disable(void)
+{
+	return lockdep_stage_test_and_disable(NULL);
+}
+
+static __always_inline void lockdep_stage_restore(unsigned long flags)
+{
+	unlock_stage(flags);
+}
+
+#else
+
+#define lockdep_stage_test_and_disable(__irqsoff)		\
+	({							\
+		unsigned long __flags;				\
+		raw_local_irq_save(__flags);			\
+		*(__irqsoff) = irqs_disabled_flags(__flags);	\
+		__flags;					\
+	})
+
+#define lockdep_stage_disable()					\
+	({							\
+		unsigned long __flags;				\
+		raw_local_irq_save(__flags);			\
+		__flags;					\
+	})
+
+#define lockdep_stage_restore(__flags)		raw_local_irq_restore(__flags)
+
+#endif /* !CONFIG_IRQ_PIPELINE */
+
 static inline void lockdep_lock(void)
 {
-	DEBUG_LOCKS_WARN_ON(!irqs_disabled());
+	DEBUG_LOCKS_WARN_ON(!hard_irqs_disabled());
 
 	__this_cpu_inc(lockdep_recursion);
 	arch_spin_lock(&__lock);
@@ -115,7 +163,7 @@ static inline void lockdep_lock(void)
 
 static inline void lockdep_unlock(void)
 {
-	DEBUG_LOCKS_WARN_ON(!irqs_disabled());
+	DEBUG_LOCKS_WARN_ON(!hard_irqs_disabled());
 
 	if (debug_locks && DEBUG_LOCKS_WARN_ON(__owner != current))
 		return;
@@ -882,7 +930,7 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
 	/*
 	 * We do an RCU walk of the hash, see lockdep_free_key_range().
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!lockdep_stage_disabled()))
 		return NULL;
 
 	hlist_for_each_entry_rcu_notrace(class, hash_head, hash_entry) {
@@ -1179,7 +1227,7 @@ void lockdep_register_key(struct lock_class_key *key)
 		return;
 	hash_head = keyhashentry(key);
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	if (!graph_lock())
 		goto restore_irqs;
 	hlist_for_each_entry_rcu(k, hash_head, hash_entry) {
@@ -1190,7 +1238,7 @@ void lockdep_register_key(struct lock_class_key *key)
 out_unlock:
 	graph_unlock();
 restore_irqs:
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lockdep_register_key);
 
@@ -1239,7 +1287,7 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
 	struct lock_class *class;
 	int idx;
 
-	DEBUG_LOCKS_WARN_ON(!irqs_disabled());
+	DEBUG_LOCKS_WARN_ON(!lockdep_stage_disabled());
 
 	class = look_up_lock_class(lock, subclass);
 	if (likely(class))
@@ -2035,11 +2083,11 @@ unsigned long lockdep_count_forward_deps(struct lock_class *class)
 
 	__bfs_init_root(&this, class);
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_lock();
 	ret = __lockdep_count_forward_deps(&this);
 	lockdep_unlock();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 
 	return ret;
 }
@@ -2061,11 +2109,11 @@ unsigned long lockdep_count_backward_deps(struct lock_class *class)
 
 	__bfs_init_root(&this, class);
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_lock();
 	ret = __lockdep_count_backward_deps(&this);
 	lockdep_unlock();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 
 	return ret;
 }
@@ -4170,6 +4218,8 @@ static void __trace_hardirqs_on_caller(void)
  */
 void lockdep_hardirqs_on_prepare(unsigned long ip)
 {
+	unsigned long flags;
+
 	if (unlikely(!debug_locks))
 		return;
 
@@ -4192,38 +4242,43 @@ void lockdep_hardirqs_on_prepare(unsigned long ip)
 		return;
 	}
 
+	flags = hard_cond_local_irq_save();
+
 	/*
 	 * We're enabling irqs and according to our state above irqs weren't
 	 * already enabled, yet we find the hardware thinks they are in fact
 	 * enabled.. someone messed up their IRQ state tracing.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-		return;
+	if (DEBUG_LOCKS_WARN_ON(!lockdep_stage_disabled()))
+		goto out;
 
 	/*
 	 * See the fine text that goes along with this variable definition.
 	 */
 	if (DEBUG_LOCKS_WARN_ON(early_boot_irqs_disabled))
-		return;
+		goto out;
 
 	/*
 	 * Can't allow enabling interrupts while in an interrupt handler,
 	 * that's general bad form and such. Recursion, limited stack etc..
 	 */
-	if (DEBUG_LOCKS_WARN_ON(lockdep_hardirq_context()))
-		return;
+	if (DEBUG_LOCKS_WARN_ON(running_inband() && lockdep_hardirq_context()))
+		goto out;
 
 	current->hardirq_chain_key = current->curr_chain_key;
 
 	lockdep_recursion_inc();
 	__trace_hardirqs_on_caller();
 	lockdep_recursion_finish();
+out:
+	hard_cond_local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lockdep_hardirqs_on_prepare);
 
 void noinstr lockdep_hardirqs_on(unsigned long ip)
 {
 	struct irqtrace_events *trace = &current->irqtrace;
+	unsigned long flags;
 
 	if (unlikely(!debug_locks))
 		return;
@@ -4261,13 +4316,15 @@ void noinstr lockdep_hardirqs_on(unsigned long ip)
 		return;
 	}
 
+	flags = hard_cond_local_irq_save();
+
 	/*
 	 * We're enabling irqs and according to our state above irqs weren't
 	 * already enabled, yet we find the hardware thinks they are in fact
 	 * enabled.. someone messed up their IRQ state tracing.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-		return;
+	if (DEBUG_LOCKS_WARN_ON(!lockdep_stage_disabled()))
+		goto out;
 
 	/*
 	 * Ensure the lock stack remained unchanged between
@@ -4282,6 +4339,8 @@ void noinstr lockdep_hardirqs_on(unsigned long ip)
 	trace->hardirq_enable_ip = ip;
 	trace->hardirq_enable_event = ++trace->irq_events;
 	debug_atomic_inc(hardirqs_on_events);
+out:
+	hard_cond_local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lockdep_hardirqs_on);
 
@@ -4290,6 +4349,8 @@ EXPORT_SYMBOL_GPL(lockdep_hardirqs_on);
  */
 void noinstr lockdep_hardirqs_off(unsigned long ip)
 {
+	unsigned long flags;
+
 	if (unlikely(!debug_locks))
 		return;
 
@@ -4304,12 +4365,14 @@ void noinstr lockdep_hardirqs_off(unsigned long ip)
 	} else if (__this_cpu_read(lockdep_recursion))
 		return;
 
+	flags = hard_cond_local_irq_save();
+
 	/*
 	 * So we're supposed to get called after you mask local IRQs, but for
 	 * some reason the hardware doesn't quite think you did a proper job.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-		return;
+	if (DEBUG_LOCKS_WARN_ON(!lockdep_stage_disabled()))
+		goto out;
 
 	if (lockdep_hardirqs_enabled()) {
 		struct irqtrace_events *trace = &current->irqtrace;
@@ -4324,6 +4387,8 @@ void noinstr lockdep_hardirqs_off(unsigned long ip)
 	} else {
 		debug_atomic_inc(redundant_hardirqs_off);
 	}
+out:
+	hard_cond_local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lockdep_hardirqs_off);
 
@@ -4333,20 +4398,23 @@ EXPORT_SYMBOL_GPL(lockdep_hardirqs_off);
 void lockdep_softirqs_on(unsigned long ip)
 {
 	struct irqtrace_events *trace = &current->irqtrace;
+	unsigned long flags;
 
 	if (unlikely(!lockdep_enabled()))
 		return;
 
+	flags = hard_cond_local_irq_save();
+
 	/*
 	 * We fancy IRQs being disabled here, see softirq.c, avoids
 	 * funny state and nesting things.
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-		return;
+	if (DEBUG_LOCKS_WARN_ON(!lockdep_stage_disabled()))
+		goto out;
 
 	if (current->softirqs_enabled) {
 		debug_atomic_inc(redundant_softirqs_on);
-		return;
+		goto out;
 	}
 
 	lockdep_recursion_inc();
@@ -4365,6 +4433,8 @@ void lockdep_softirqs_on(unsigned long ip)
 	if (lockdep_hardirqs_enabled())
 		mark_held_locks(current, LOCK_ENABLED_SOFTIRQ);
 	lockdep_recursion_finish();
+out:
+	hard_cond_local_irq_restore(flags);
 }
 
 /*
@@ -4372,14 +4442,18 @@ void lockdep_softirqs_on(unsigned long ip)
  */
 void lockdep_softirqs_off(unsigned long ip)
 {
+	unsigned long flags;
+
 	if (unlikely(!lockdep_enabled()))
 		return;
 
+	flags = hard_cond_local_irq_save();
+
 	/*
 	 * We fancy IRQs being disabled here, see softirq.c
 	 */
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
-		return;
+	if (DEBUG_LOCKS_WARN_ON(!lockdep_stage_disabled()))
+		goto out;
 
 	if (current->softirqs_enabled) {
 		struct irqtrace_events *trace = &current->irqtrace;
@@ -4397,6 +4471,8 @@ void lockdep_softirqs_off(unsigned long ip)
 		DEBUG_LOCKS_WARN_ON(!softirq_count());
 	} else
 		debug_atomic_inc(redundant_softirqs_off);
+out:
+	hard_cond_local_irq_restore(flags);
 }
 
 static int
@@ -4751,11 +4827,11 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name,
 		if (DEBUG_LOCKS_WARN_ON(!lockdep_enabled()))
 			return;
 
-		raw_local_irq_save(flags);
+		flags = lockdep_stage_disable();
 		lockdep_recursion_inc();
 		register_lock_class(lock, subclass, 1);
 		lockdep_recursion_finish();
-		raw_local_irq_restore(flags);
+		lockdep_stage_restore(flags);
 	}
 }
 EXPORT_SYMBOL_GPL(lockdep_init_map_type);
@@ -5085,7 +5161,7 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth,
 	struct held_lock *hlock;
 	int first_idx = idx;
 
-	if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
+	if (DEBUG_LOCKS_WARN_ON(!lockdep_stage_disabled()))
 		return 0;
 
 	for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) {
@@ -5397,7 +5473,13 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie
 static noinstr void check_flags(unsigned long flags)
 {
 #if defined(CONFIG_PROVE_LOCKING) && defined(CONFIG_DEBUG_LOCKDEP)
-	if (!debug_locks)
+	/*
+	 * irq_pipeline: we can't and don't want to check the
+	 * consistency of the irq tracer when running over the
+	 * pipeline entry or oob stage contexts, since the inband
+	 * stall bit does not reflect the current irq state there.
+	 */
+	if (on_pipeline_entry() || running_oob() || !debug_locks)
 		return;
 
 	/* Get the warning out..  */
@@ -5444,13 +5526,13 @@ void lock_set_class(struct lockdep_map *lock, const char *name,
 	if (unlikely(!lockdep_enabled()))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_recursion_inc();
 	check_flags(flags);
 	if (__lock_set_class(lock, name, key, subclass, ip))
 		check_chain_key(current);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_set_class);
 
@@ -5461,13 +5543,13 @@ void lock_downgrade(struct lockdep_map *lock, unsigned long ip)
 	if (unlikely(!lockdep_enabled()))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_recursion_inc();
 	check_flags(flags);
 	if (__lock_downgrade(lock, ip))
 		check_chain_key(current);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_downgrade);
 
@@ -5532,6 +5614,7 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 			  struct lockdep_map *nest_lock, unsigned long ip)
 {
 	unsigned long flags;
+	int irqsoff;
 
 	trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
 
@@ -5558,14 +5641,14 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 		return;
 	}
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_test_and_disable(&irqsoff);
 	check_flags(flags);
 
 	lockdep_recursion_inc();
 	__lock_acquire(lock, subclass, trylock, read, check,
-		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0);
+		       irqsoff, nest_lock, ip, 0, 0);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_acquire);
 
@@ -5578,14 +5661,14 @@ void lock_release(struct lockdep_map *lock, unsigned long ip)
 	if (unlikely(!lockdep_enabled()))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	check_flags(flags);
 
 	lockdep_recursion_inc();
 	if (__lock_release(lock, ip))
 		check_chain_key(current);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_release);
 
@@ -5597,13 +5680,13 @@ noinstr int lock_is_held_type(const struct lockdep_map *lock, int read)
 	if (unlikely(!lockdep_enabled()))
 		return 1; /* avoid false negative lockdep_assert_held() */
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	check_flags(flags);
 
 	lockdep_recursion_inc();
 	ret = __lock_is_held(lock, read);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 
 	return ret;
 }
@@ -5618,13 +5701,13 @@ struct pin_cookie lock_pin_lock(struct lockdep_map *lock)
 	if (unlikely(!lockdep_enabled()))
 		return cookie;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	check_flags(flags);
 
 	lockdep_recursion_inc();
 	cookie = __lock_pin_lock(lock);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 
 	return cookie;
 }
@@ -5637,13 +5720,13 @@ void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
 	if (unlikely(!lockdep_enabled()))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	check_flags(flags);
 
 	lockdep_recursion_inc();
 	__lock_repin_lock(lock, cookie);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_repin_lock);
 
@@ -5654,13 +5737,13 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
 	if (unlikely(!lockdep_enabled()))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	check_flags(flags);
 
 	lockdep_recursion_inc();
 	__lock_unpin_lock(lock, cookie);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_unpin_lock);
 
@@ -5790,12 +5873,12 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip)
 	if (unlikely(!lock_stat || !lockdep_enabled()))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	check_flags(flags);
 	lockdep_recursion_inc();
 	__lock_contended(lock, ip);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_contended);
 
@@ -5808,12 +5891,12 @@ void lock_acquired(struct lockdep_map *lock, unsigned long ip)
 	if (unlikely(!lock_stat || !lockdep_enabled()))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	check_flags(flags);
 	lockdep_recursion_inc();
 	__lock_acquired(lock, ip);
 	lockdep_recursion_finish();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_acquired);
 #endif
@@ -5828,7 +5911,7 @@ void lockdep_reset(void)
 	unsigned long flags;
 	int i;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_init_task(current);
 	memset(current->held_locks, 0, MAX_LOCK_DEPTH*sizeof(struct held_lock));
 	nr_hardirq_chains = 0;
@@ -5837,7 +5920,7 @@ void lockdep_reset(void)
 	debug_locks = 1;
 	for (i = 0; i < CHAINHASH_SIZE; i++)
 		INIT_HLIST_HEAD(chainhash_table + i);
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 
 /* Remove a class from a lock chain. Must be called with the graph lock held. */
@@ -6014,7 +6097,7 @@ static void free_zapped_rcu(struct rcu_head *ch)
 	if (WARN_ON_ONCE(ch != &delayed_free.rcu_head))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_lock();
 
 	/* closed head */
@@ -6028,7 +6111,7 @@ static void free_zapped_rcu(struct rcu_head *ch)
 	call_rcu_zapped(delayed_free.pf + delayed_free.index);
 
 	lockdep_unlock();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 
 /*
@@ -6071,13 +6154,13 @@ static void lockdep_free_key_range_reg(void *start, unsigned long size)
 
 	init_data_structures_once();
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_lock();
 	pf = get_pending_free();
 	__lockdep_free_key_range(pf, start, size);
 	call_rcu_zapped(pf);
 	lockdep_unlock();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 
 	/*
 	 * Wait for any possible iterators from look_up_lock_class() to pass
@@ -6097,12 +6180,12 @@ static void lockdep_free_key_range_imm(void *start, unsigned long size)
 
 	init_data_structures_once();
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_lock();
 	__lockdep_free_key_range(pf, start, size);
 	__free_zapped_classes(pf);
 	lockdep_unlock();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 
 void lockdep_free_key_range(void *start, unsigned long size)
@@ -6173,7 +6256,7 @@ static void lockdep_reset_lock_reg(struct lockdep_map *lock)
 	unsigned long flags;
 	int locked;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	locked = graph_lock();
 	if (!locked)
 		goto out_irq;
@@ -6184,7 +6267,7 @@ static void lockdep_reset_lock_reg(struct lockdep_map *lock)
 
 	graph_unlock();
 out_irq:
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 
 /*
@@ -6196,12 +6279,12 @@ static void lockdep_reset_lock_imm(struct lockdep_map *lock)
 	struct pending_free *pf = delayed_free.pf;
 	unsigned long flags;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_lock();
 	__lockdep_reset_lock(pf, lock);
 	__free_zapped_classes(pf);
 	lockdep_unlock();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 
 void lockdep_reset_lock(struct lockdep_map *lock)
@@ -6234,7 +6317,7 @@ void lockdep_unregister_key(struct lock_class_key *key)
 	if (WARN_ON_ONCE(static_obj(key)))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	lockdep_lock();
 
 	hlist_for_each_entry_rcu(k, hash_head, hash_entry) {
@@ -6251,7 +6334,7 @@ void lockdep_unregister_key(struct lock_class_key *key)
 		call_rcu_zapped(pf);
 	}
 	lockdep_unlock();
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 
 	/* Wait until is_dynamic_key() has finished accessing k->hash_entry. */
 	synchronize_rcu();
@@ -6342,7 +6425,7 @@ void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len)
 	if (unlikely(!debug_locks))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = lockdep_stage_disable();
 	for (i = 0; i < curr->lockdep_depth; i++) {
 		hlock = curr->held_locks + i;
 
@@ -6353,7 +6436,7 @@ void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len)
 		print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock);
 		break;
 	}
-	raw_local_irq_restore(flags);
+	lockdep_stage_restore(flags);
 }
 EXPORT_SYMBOL_GPL(debug_check_no_locks_freed);
 
diff --git a/kernel/kernel/locking/lockdep_internals.h b/kernel/kernel/locking/lockdep_internals.h
index bbe9000260..6f78acc29b 100644
--- a/kernel/kernel/locking/lockdep_internals.h
+++ b/kernel/kernel/locking/lockdep_internals.h
@@ -213,12 +213,12 @@ DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats);
 	this_cpu_inc(lockdep_stats.ptr);
 
 #define debug_atomic_inc(ptr)			{		\
-	WARN_ON_ONCE(!irqs_disabled());				\
+	WARN_ON_ONCE(!hard_irqs_disabled() && !irqs_disabled());\
 	__this_cpu_inc(lockdep_stats.ptr);			\
 }
 
 #define debug_atomic_dec(ptr)			{		\
-	WARN_ON_ONCE(!irqs_disabled());				\
+	WARN_ON_ONCE(!hard_irqs_disabled() && !irqs_disabled());\
 	__this_cpu_dec(lockdep_stats.ptr);			\
 }
 
diff --git a/kernel/kernel/locking/pipeline.c b/kernel/kernel/locking/pipeline.c
new file mode 100644
index 0000000000..fde458e82a
--- /dev/null
+++ b/kernel/kernel/locking/pipeline.c
@@ -0,0 +1,231 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2016 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#include <linux/linkage.h>
+#include <linux/preempt.h>
+#include <linux/spinlock.h>
+#include <linux/interrupt.h>
+#include <linux/irq_pipeline.h>
+#include <linux/kconfig.h>
+
+/*
+ * A hybrid spinlock behaves in different ways depending on the
+ * current interrupt stage on entry.
+ *
+ * Such spinlock always leaves hard IRQs disabled once locked. In
+ * addition, it stalls the in-band stage when protecting a critical
+ * section there, disabling preemption like regular spinlocks do as
+ * well. This combination preserves the regular locking logic when
+ * called from the in-band stage, while fully disabling preemption by
+ * other interrupt stages.
+ *
+ * When taken from the pipeline entry context, a hybrid lock behaves
+ * like a hard spinlock, assuming that hard IRQs are already disabled.
+ *
+ * The irq descriptor lock (struct irq_desc) is a typical example of
+ * such lock, which properly serializes accesses regardless of the
+ * calling context.
+ */
+void __hybrid_spin_lock(struct raw_spinlock *rlock)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags;
+
+	if (running_inband())
+		preempt_disable();
+
+	__flags = hard_local_irq_save();
+	hard_lock_acquire(rlock, 0, _RET_IP_);
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	lock->hwflags = __flags;
+}
+EXPORT_SYMBOL(__hybrid_spin_lock);
+
+void __hybrid_spin_lock_nested(struct raw_spinlock *rlock, int subclass)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags;
+
+	if (running_inband())
+		preempt_disable();
+
+	__flags = hard_local_irq_save();
+	hard_lock_acquire_nested(rlock, subclass, _RET_IP_);
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	lock->hwflags = __flags;
+}
+EXPORT_SYMBOL(__hybrid_spin_lock_nested);
+
+void __hybrid_spin_unlock(struct raw_spinlock *rlock)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags;
+
+	/* Pick the flags before releasing the lock. */
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	__flags = lock->hwflags;
+	hard_lock_release(rlock, _RET_IP_);
+	hard_local_irq_restore(__flags);
+
+	if (running_inband())
+		preempt_enable();
+}
+EXPORT_SYMBOL(__hybrid_spin_unlock);
+
+void __hybrid_spin_lock_irq(struct raw_spinlock *rlock)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags;
+
+	__flags = hard_local_irq_save();
+
+	if (running_inband()) {
+		stall_inband();
+		trace_hardirqs_off();
+		preempt_disable();
+	}
+
+	hard_lock_acquire(rlock, 0, _RET_IP_);
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	lock->hwflags = __flags;
+}
+EXPORT_SYMBOL(__hybrid_spin_lock_irq);
+
+void __hybrid_spin_unlock_irq(struct raw_spinlock *rlock)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags;
+
+	/* Pick the flags before releasing the lock. */
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	__flags = lock->hwflags;
+	hard_lock_release(rlock, _RET_IP_);
+
+	if (running_inband()) {
+		trace_hardirqs_on();
+		unstall_inband_nocheck();
+		hard_local_irq_restore(__flags);
+		preempt_enable();
+		return;
+	}
+
+	hard_local_irq_restore(__flags);
+}
+EXPORT_SYMBOL(__hybrid_spin_unlock_irq);
+
+unsigned long __hybrid_spin_lock_irqsave(struct raw_spinlock *rlock)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags, flags;
+
+	__flags = flags = hard_local_irq_save();
+
+	if (running_inband()) {
+		flags = test_and_stall_inband();
+		trace_hardirqs_off();
+		preempt_disable();
+	}
+
+	hard_lock_acquire(rlock, 0, _RET_IP_);
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	lock->hwflags = __flags;
+
+	return flags;
+}
+EXPORT_SYMBOL(__hybrid_spin_lock_irqsave);
+
+void __hybrid_spin_unlock_irqrestore(struct raw_spinlock *rlock,
+				      unsigned long flags)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags;
+
+	/* Pick the flags before releasing the lock. */
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	__flags = lock->hwflags;
+	hard_lock_release(rlock, _RET_IP_);
+
+	if (running_inband()) {
+		if (!flags) {
+			trace_hardirqs_on();
+			unstall_inband_nocheck();
+		}
+		hard_local_irq_restore(__flags);
+		preempt_enable();
+		return;
+	}
+
+	hard_local_irq_restore(__flags);
+}
+EXPORT_SYMBOL(__hybrid_spin_unlock_irqrestore);
+
+int __hybrid_spin_trylock(struct raw_spinlock *rlock)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags;
+
+	if (running_inband())
+		preempt_disable();
+
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	__flags = hard_local_irq_save();
+
+	hard_spin_trylock_prepare(rlock);
+	if (do_raw_spin_trylock(rlock)) {
+		lock->hwflags = __flags;
+		hard_trylock_acquire(rlock, 1, _RET_IP_);
+		return 1;
+	}
+
+	hard_spin_trylock_fail(rlock);
+	hard_local_irq_restore(__flags);
+
+	if (running_inband())
+		preempt_enable();
+
+	return 0;
+}
+EXPORT_SYMBOL(__hybrid_spin_trylock);
+
+int __hybrid_spin_trylock_irqsave(struct raw_spinlock *rlock,
+				   unsigned long *flags)
+{
+	struct hybrid_spinlock *lock;
+	unsigned long __flags;
+	bool inband;
+
+	inband = running_inband();
+
+	__flags = *flags = hard_local_irq_save();
+
+	lock = container_of(rlock, struct hybrid_spinlock, rlock);
+	if (inband) {
+		*flags = test_and_stall_inband();
+		trace_hardirqs_off();
+		preempt_disable();
+	}
+
+	hard_spin_trylock_prepare(rlock);
+	if (do_raw_spin_trylock(rlock)) {
+		hard_trylock_acquire(rlock, 1, _RET_IP_);
+		lock->hwflags = __flags;
+		return 1;
+	}
+
+	hard_spin_trylock_fail(rlock);
+
+	if (inband && !*flags) {
+		trace_hardirqs_on();
+		unstall_inband_nocheck();
+	}
+
+	hard_local_irq_restore(__flags);
+
+	if (inband)
+		preempt_enable();
+
+	return 0;
+}
+EXPORT_SYMBOL(__hybrid_spin_trylock_irqsave);
diff --git a/kernel/kernel/locking/spinlock_debug.c b/kernel/kernel/locking/spinlock_debug.c
index b9d93087ee..0bcc39f5b1 100644
--- a/kernel/kernel/locking/spinlock_debug.c
+++ b/kernel/kernel/locking/spinlock_debug.c
@@ -114,6 +114,7 @@ void do_raw_spin_lock(raw_spinlock_t *lock)
 	mmiowb_spin_lock();
 	debug_spin_lock_after(lock);
 }
+EXPORT_SYMBOL_GPL(do_raw_spin_lock);
 
 int do_raw_spin_trylock(raw_spinlock_t *lock)
 {
@@ -131,6 +132,7 @@ int do_raw_spin_trylock(raw_spinlock_t *lock)
 #endif
 	return ret;
 }
+EXPORT_SYMBOL_GPL(do_raw_spin_trylock);
 
 void do_raw_spin_unlock(raw_spinlock_t *lock)
 {
@@ -138,6 +140,7 @@ void do_raw_spin_unlock(raw_spinlock_t *lock)
 	debug_spin_unlock(lock);
 	arch_spin_unlock(&lock->raw_lock);
 }
+EXPORT_SYMBOL_GPL(do_raw_spin_unlock);
 
 static void rwlock_bug(rwlock_t *lock, const char *msg)
 {
diff --git a/kernel/kernel/notifier.c b/kernel/kernel/notifier.c
index 1b019cbca5..b116e1477e 100644
--- a/kernel/kernel/notifier.c
+++ b/kernel/kernel/notifier.c
@@ -213,6 +213,9 @@ int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
 {
 	int ret;
 
+	if (!running_inband())
+		return notifier_call_chain(&nh->head, val, v, -1, NULL);
+
 	rcu_read_lock();
 	ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
 	rcu_read_unlock();
diff --git a/kernel/kernel/panic.c b/kernel/kernel/panic.c
index 332736a72a..f4a05dd5a6 100644
--- a/kernel/kernel/panic.c
+++ b/kernel/kernel/panic.c
@@ -27,6 +27,7 @@
 #include <linux/sysrq.h>
 #include <linux/init.h>
 #include <linux/nmi.h>
+#include <linux/irq_pipeline.h>
 #include <linux/console.h>
 #include <linux/bug.h>
 #include <linux/ratelimit.h>
@@ -49,7 +50,7 @@ static unsigned long tainted_mask =
 	IS_ENABLED(CONFIG_GCC_PLUGIN_RANDSTRUCT) ? (1 << TAINT_RANDSTRUCT) : 0;
 static int pause_on_oops;
 static int pause_on_oops_flag;
-static DEFINE_SPINLOCK(pause_on_oops_lock);
+static DEFINE_HARD_SPINLOCK(pause_on_oops_lock);
 bool crash_kexec_post_notifiers;
 int panic_on_warn __read_mostly;
 unsigned long panic_on_taint;
@@ -189,8 +190,9 @@ void panic(const char *fmt, ...)
 	 * there is nothing to prevent an interrupt handler (that runs
 	 * after setting panic_cpu) from invoking panic() again.
 	 */
-	local_irq_disable();
+	hard_local_irq_disable();
 	preempt_disable_notrace();
+	irq_pipeline_oops();
 
 	/*
 	 * It's possible to come here directly from a panic-assertion and
@@ -267,9 +269,12 @@ void panic(const char *fmt, ...)
 
 	/*
 	 * Run any panic handlers, including those that might need to
-	 * add information to the kmsg dump output.
+	 * add information to the kmsg dump output. Skip panic
+	 * handlers if running over the oob stage, as they would most
+	 * likely break.
 	 */
-	atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
+	if (running_inband())
+		atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
 
 	/* Call flush even twice. It tries harder with a single online CPU */
 	printk_safe_flush_on_panic();
@@ -474,7 +479,7 @@ static void do_oops_enter_exit(void)
 	if (!pause_on_oops)
 		return;
 
-	spin_lock_irqsave(&pause_on_oops_lock, flags);
+	raw_spin_lock_irqsave(&pause_on_oops_lock, flags);
 	if (pause_on_oops_flag == 0) {
 		/* This CPU may now print the oops message */
 		pause_on_oops_flag = 1;
@@ -484,21 +489,21 @@ static void do_oops_enter_exit(void)
 			/* This CPU gets to do the counting */
 			spin_counter = pause_on_oops;
 			do {
-				spin_unlock(&pause_on_oops_lock);
+				raw_spin_unlock(&pause_on_oops_lock);
 				spin_msec(MSEC_PER_SEC);
-				spin_lock(&pause_on_oops_lock);
+				raw_spin_lock(&pause_on_oops_lock);
 			} while (--spin_counter);
 			pause_on_oops_flag = 0;
 		} else {
 			/* This CPU waits for a different one */
 			while (spin_counter) {
-				spin_unlock(&pause_on_oops_lock);
+				raw_spin_unlock(&pause_on_oops_lock);
 				spin_msec(1);
-				spin_lock(&pause_on_oops_lock);
+				raw_spin_lock(&pause_on_oops_lock);
 			}
 		}
 	}
-	spin_unlock_irqrestore(&pause_on_oops_lock, flags);
+	raw_spin_unlock_irqrestore(&pause_on_oops_lock, flags);
 }
 
 /*
@@ -528,6 +533,7 @@ void oops_enter(void)
 {
 	tracing_off();
 	/* can't trust the integrity of the kernel anymore: */
+	irq_pipeline_oops();
 	debug_locks_off();
 	do_oops_enter_exit();
 
diff --git a/kernel/kernel/power/Makefile b/kernel/kernel/power/Makefile
index 97705757f9..bab122142b 100644
--- a/kernel/kernel/power/Makefile
+++ b/kernel/kernel/power/Makefile
@@ -17,5 +17,7 @@ obj-$(CONFIG_PM_WAKELOCKS)	+= wakelock.o
 
 obj-$(CONFIG_MAGIC_SYSRQ)	+= poweroff.o
 
+ifndef CONFIG_DOVETAIL
 obj-$(CONFIG_SUSPEND)		+= wakeup_reason.o
+endif
 obj-$(CONFIG_ENERGY_MODEL)	+= energy_model.o
diff --git a/kernel/kernel/power/hibernate.c b/kernel/kernel/power/hibernate.c
index b6875ebd18..bc68e0f156 100644
--- a/kernel/kernel/power/hibernate.c
+++ b/kernel/kernel/power/hibernate.c
@@ -302,6 +302,7 @@ static int create_image(int platform_mode)
 		goto Enable_cpus;
 
 	local_irq_disable();
+	hard_cond_local_irq_disable();
 
 	system_state = SYSTEM_SUSPEND;
 
@@ -469,6 +470,7 @@ static int resume_target_kernel(bool platform_mode)
 
 	local_irq_disable();
 	system_state = SYSTEM_SUSPEND;
+	hard_cond_local_irq_disable();
 
 	error = syscore_suspend();
 	if (error)
@@ -590,6 +592,7 @@ int hibernation_platform_enter(void)
 
 	local_irq_disable();
 	system_state = SYSTEM_SUSPEND;
+	hard_cond_local_irq_disable();
 	syscore_suspend();
 	if (pm_wakeup_pending()) {
 		error = -EAGAIN;
diff --git a/kernel/kernel/printk/printk.c b/kernel/kernel/printk/printk.c
index e2534757a4..db67ef3281 100644
--- a/kernel/kernel/printk/printk.c
+++ b/kernel/kernel/printk/printk.c
@@ -47,6 +47,7 @@
 #include <linux/sched/clock.h>
 #include <linux/sched/debug.h>
 #include <linux/sched/task_stack.h>
+#include <linux/irqstage.h>
 
 #include <linux/uaccess.h>
 #include <asm/sections.h>
@@ -2188,6 +2189,73 @@ asmlinkage __visible void early_printk(const char *fmt, ...)
 }
 #endif
 
+#ifdef CONFIG_RAW_PRINTK
+static struct console *raw_console;
+static DEFINE_HARD_SPINLOCK(raw_console_lock);
+
+void raw_puts(const char *s, size_t len)
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&raw_console_lock, flags);
+	if (raw_console)
+		raw_console->write_raw(raw_console, s, len);
+	raw_spin_unlock_irqrestore(&raw_console_lock, flags);
+}
+EXPORT_SYMBOL(raw_puts);
+
+void raw_vprintk(const char *fmt, va_list ap)
+{
+	char buf[256];
+	size_t n;
+
+	if (raw_console == NULL || console_suspended)
+		return;
+
+        touch_nmi_watchdog();
+	n = vscnprintf(buf, sizeof(buf), fmt, ap);
+	raw_puts(buf, n);
+}
+EXPORT_SYMBOL(raw_vprintk);
+
+asmlinkage __visible void raw_printk(const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	raw_vprintk(fmt, ap);
+	va_end(ap);
+}
+EXPORT_SYMBOL(raw_printk);
+
+static inline void register_raw_console(struct console *newcon)
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&raw_console_lock, flags);
+	if (newcon->write_raw)
+		raw_console = newcon;
+	raw_spin_unlock_irqrestore(&raw_console_lock, flags);
+}
+
+static inline void unregister_raw_console(struct console *oldcon)
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&raw_console_lock, flags);
+	if (oldcon == raw_console)
+		raw_console = NULL;
+	raw_spin_unlock_irqrestore(&raw_console_lock, flags);
+}
+
+#else
+
+static inline void register_raw_console(struct console *newcon) { }
+
+static inline void unregister_raw_console(struct console *oldcon) { }
+
+#endif
+
 static int __add_preferred_console(char *name, int idx, char *options,
 				   char *brl_options, bool user_specified)
 {
@@ -2854,6 +2922,9 @@ void register_console(struct console *newcon)
 	if (err || newcon->flags & CON_BRL)
 		return;
 
+	/* The latest raw console to register is current. */
+	register_raw_console(newcon);
+
 	/*
 	 * If we have a bootconsole, and are switching to a real console,
 	 * don't print everything out again, since when the boot console, and
@@ -2938,6 +3009,8 @@ int unregister_console(struct console *console)
 		(console->flags & CON_BOOT) ? "boot" : "" ,
 		console->name, console->index);
 
+	unregister_raw_console(console);
+
 	res = _braille_unregister_console(console);
 	if (res < 0)
 		return res;
diff --git a/kernel/kernel/printk/printk_safe.c b/kernel/kernel/printk/printk_safe.c
index 2e9e3ed7d6..9c065e42de 100644
--- a/kernel/kernel/printk/printk_safe.c
+++ b/kernel/kernel/printk/printk_safe.c
@@ -9,6 +9,7 @@
 #include <linux/kdb.h>
 #include <linux/smp.h>
 #include <linux/cpumask.h>
+#include <linux/irq_pipeline.h>
 #include <linux/irq_work.h>
 #include <linux/printk.h>
 #include <linux/kprobes.h>
@@ -374,6 +375,8 @@ __printf(1, 0) int vprintk_func(const char *fmt, va_list args)
 	if (unlikely(kdb_trap_printk && kdb_printf_cpu < 0))
 		return vkdb_printf(KDB_MSGSRC_PRINTK, fmt, args);
 #endif
+	if (inband_unsafe())
+		return vprintk_nmi(fmt, args);
 
 	/*
 	 * Try to use the main logbuf even in NMI. But avoid calling console
diff --git a/kernel/kernel/ptrace.c b/kernel/kernel/ptrace.c
index aab480e24b..18f26ae0df 100644
--- a/kernel/kernel/ptrace.c
+++ b/kernel/kernel/ptrace.c
@@ -854,10 +854,12 @@ static int ptrace_resume(struct task_struct *child, long request,
 		if (unlikely(!arch_has_block_step()))
 			return -EIO;
 		user_enable_block_step(child);
+		inband_ptstep_notify(child);
 	} else if (is_singlestep(request) || is_sysemu_singlestep(request)) {
 		if (unlikely(!arch_has_single_step()))
 			return -EIO;
 		user_enable_single_step(child);
+		inband_ptstep_notify(child);
 	} else {
 		user_disable_single_step(child);
 	}
diff --git a/kernel/kernel/rcu/tree.c b/kernel/kernel/rcu/tree.c
index b10d6bcea7..7e71bc83be 100644
--- a/kernel/kernel/rcu/tree.c
+++ b/kernel/kernel/rcu/tree.c
@@ -232,6 +232,11 @@ static long rcu_get_n_cbs_cpu(int cpu)
 	return 0;
 }
 
+static inline bool rcu_in_nonmaskable(void)
+{
+	return on_pipeline_entry() || in_nmi();
+}
+
 void rcu_softirq_qs(void)
 {
 	rcu_qs();
@@ -710,6 +715,7 @@ noinstr void rcu_nmi_exit(void)
 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
 
 	instrumentation_begin();
+
 	/*
 	 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
 	 * (We are exiting an NMI handler, so RCU better be paying attention
@@ -735,7 +741,7 @@ noinstr void rcu_nmi_exit(void)
 	trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
 	WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
 
-	if (!in_nmi())
+	if (!rcu_in_nonmaskable())
 		rcu_prepare_for_idle();
 
 	// instrumentation for the noinstr rcu_dynticks_eqs_enter()
@@ -746,7 +752,7 @@ noinstr void rcu_nmi_exit(void)
 	rcu_dynticks_eqs_enter();
 	// ... but is no longer watching here.
 
-	if (!in_nmi())
+	if (!rcu_in_nonmaskable())
 		rcu_dynticks_task_enter();
 }
 
@@ -935,7 +941,7 @@ void __rcu_irq_enter_check_tick(void)
 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
 
 	// If we're here from NMI there's nothing to do.
-	if (in_nmi())
+	if (rcu_in_nonmaskable())
 		return;
 
 	RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
@@ -996,14 +1002,14 @@ noinstr void rcu_nmi_enter(void)
 	 */
 	if (rcu_dynticks_curr_cpu_in_eqs()) {
 
-		if (!in_nmi())
+		if (!rcu_in_nonmaskable())
 			rcu_dynticks_task_exit();
 
 		// RCU is not watching here ...
 		rcu_dynticks_eqs_exit();
 		// ... but is watching here.
 
-		if (!in_nmi()) {
+		if (!rcu_in_nonmaskable()) {
 			instrumentation_begin();
 			rcu_cleanup_after_idle();
 			instrumentation_end();
@@ -1016,7 +1022,7 @@ noinstr void rcu_nmi_enter(void)
 		instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
 
 		incby = 1;
-	} else if (!in_nmi()) {
+	} else if (!rcu_in_nonmaskable()) {
 		instrumentation_begin();
 		rcu_irq_enter_check_tick();
 	} else  {
@@ -1094,10 +1100,10 @@ static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp)
 /**
  * rcu_is_watching - see if RCU thinks that the current CPU is not idle
  *
- * Return true if RCU is watching the running CPU, which means that this
- * CPU can safely enter RCU read-side critical sections.  In other words,
- * if the current CPU is not in its idle loop or is in an interrupt or
- * NMI handler, return true.
+ * Return true if RCU is watching the running CPU, which means that
+ * this CPU can safely enter RCU read-side critical sections.  In
+ * other words, if the current CPU is not in its idle loop or is in an
+ * interrupt or NMI handler, return true.
  *
  * Make notrace because it can be called by the internal functions of
  * ftrace, and making this notrace removes unnecessary recursion calls.
@@ -1106,6 +1112,9 @@ notrace bool rcu_is_watching(void)
 {
 	bool ret;
 
+	if (on_pipeline_entry())
+ 		return true;
+ 
 	preempt_disable_notrace();
 	ret = !rcu_dynticks_curr_cpu_in_eqs();
 	preempt_enable_notrace();
@@ -1152,7 +1161,7 @@ bool rcu_lockdep_current_cpu_online(void)
 	struct rcu_node *rnp;
 	bool ret = false;
 
-	if (in_nmi() || !rcu_scheduler_fully_active)
+	if (rcu_in_nonmaskable() || !rcu_scheduler_fully_active)
 		return true;
 	preempt_disable_notrace();
 	rdp = this_cpu_ptr(&rcu_data);
diff --git a/kernel/kernel/rcu/tree_plugin.h b/kernel/kernel/rcu/tree_plugin.h
index f5ba0740f9..7d9ec7a0e1 100644
--- a/kernel/kernel/rcu/tree_plugin.h
+++ b/kernel/kernel/rcu/tree_plugin.h
@@ -790,7 +790,7 @@ void rcu_read_unlock_strict(void)
 	struct rcu_data *rdp;
 
 	if (!IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ||
-	   irqs_disabled() || preempt_count() || !rcu_state.gp_kthread)
+		on_pipeline_entry() || running_oob() || irqs_disabled() || preempt_count() || !rcu_state.gp_kthread)
 		return;
 	rdp = this_cpu_ptr(&rcu_data);
 	rcu_report_qs_rdp(rdp);
diff --git a/kernel/kernel/rcu/update.c b/kernel/kernel/rcu/update.c
index 849f0aa993..8e4b48bee1 100644
--- a/kernel/kernel/rcu/update.c
+++ b/kernel/kernel/rcu/update.c
@@ -99,6 +99,11 @@ module_param(rcu_normal_after_boot, int, 0);
  */
 static bool rcu_read_lock_held_common(bool *ret)
 {
+	if (irqs_pipelined() &&
+		(hard_irqs_disabled() || running_oob())) {
+		*ret = 1;
+		return true;
+	}
 	if (!debug_lockdep_rcu_enabled()) {
 		*ret = true;
 		return true;
@@ -209,6 +214,32 @@ EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended);
 
 #endif /* #ifndef CONFIG_TINY_RCU */
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+/*
+ * Prepare for taking the RCU read lock when running out-of-band. Nop
+ * otherwise.
+ */
+void rcu_oob_prepare_lock(void)
+{
+	if (!on_pipeline_entry() && running_oob())
+		rcu_nmi_enter();
+}
+EXPORT_SYMBOL_GPL(rcu_oob_prepare_lock);
+
+/*
+ * Converse to rcu_oob_prepare_lock(), after dropping the RCU read
+ * lock.
+ */
+void rcu_oob_finish_lock(void)
+{
+	if (!on_pipeline_entry() && running_oob())
+		rcu_nmi_exit();
+}
+EXPORT_SYMBOL_GPL(rcu_oob_finish_lock);
+
+#endif	/* CONFIG_IRQ_PIPELINE */
+
 /*
  * Test each non-SRCU synchronous grace-period wait API.  This is
  * useful just after a change in mode for these primitives, and
diff --git a/kernel/kernel/sched/core.c b/kernel/kernel/sched/core.c
index 73593759dd..b14a6fbdd8 100644
--- a/kernel/kernel/sched/core.c
+++ b/kernel/kernel/sched/core.c
@@ -2045,6 +2045,7 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
 	if (cpumask_test_cpu(task_cpu(p), new_mask))
 		goto out;
 
+	inband_migration_notify(p, dest_cpu);
 	if (task_running(rq, p) || p->state == TASK_WAKING) {
 		struct migration_arg arg = { p, dest_cpu };
 		/* Need help from migration thread: drop lock and wait. */
@@ -3065,7 +3066,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 		 *  - we're serialized against set_special_state() by virtue of
 		 *    it disabling IRQs (this allows not taking ->pi_lock).
 		 */
-		if (!(p->state & state))
+		if (!(p->state & state) || task_is_off_stage(p))
 			goto out;
 
 		success = 1;
@@ -3083,7 +3084,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 	 */
 	raw_spin_lock_irqsave(&p->pi_lock, flags);
 	smp_mb__after_spinlock();
-	if (!(p->state & state))
+	if (!(p->state & state) || task_is_off_stage(p))
 		goto unlock;
 
 #ifdef CONFIG_FREEZER
@@ -3349,6 +3350,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
 #ifdef CONFIG_SMP
 	p->wake_entry.u_flags = CSD_TYPE_TTWU;
 #endif
+#ifdef CONFIG_IRQ_PIPELINE
+	init_task_stall_bits(p);
+#endif
 }
 
 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -3816,6 +3820,13 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,
 	rseq_preempt(prev);
 	fire_sched_out_preempt_notifiers(prev, next);
 	prepare_task(next);
+	prepare_inband_switch(next);
+	/*
+	 * Do not fold the following hard irqs disabling into
+	 * prepare_inband_switch(), this is required when pipelining
+	 * interrupts, not only by alternate scheduling.
+	 */
+	hard_cond_local_irq_disable();
 	prepare_arch_switch(next);
 }
 
@@ -3973,8 +3984,19 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev)
 	 * finish_task_switch() will drop rq->lock() and lower preempt_count
 	 * and the preempt_enable() will end up enabling preemption (on
 	 * PREEMPT_COUNT kernels).
-	 */
-
+	 *
+	 * If interrupts are pipelined, we may enable hard irqs since
+	 * the in-band stage is stalled. If dovetailing is enabled
+	 * too, schedule_tail() is the place where transitions of
+	 * tasks from the in-band to the oob stage completes. The
+	 * companion core is notified that 'prev' is now suspended in
+	 * the in-band stage, and can be safely resumed in the oob
+	 * stage.
+	 */
+
+	WARN_ON_ONCE(irq_pipeline_debug() && !irqs_disabled());
+	hard_cond_local_irq_enable();
+	oob_trampoline();
 	rq = finish_task_switch(prev);
 	balance_callback(rq);
 	preempt_enable();
@@ -4028,6 +4050,20 @@ context_switch(struct rq *rq, struct task_struct *prev,
 		 */
 		switch_mm_irqs_off(prev->active_mm, next->mm, next);
 
+		/*
+		 * If dovetail is enabled, insert a short window of
+		 * opportunity for preemption by out-of-band IRQs
+		 * before finalizing the context switch.
+		 * dovetail_context_switch() can deal with preempting
+		 * partially switched in-band contexts.
+		 */
+		if (dovetailing()) {
+			struct mm_struct *oldmm = prev->active_mm;
+			prev->active_mm = next->mm;
+			hard_local_irq_sync();
+			prev->active_mm = oldmm;
+		}
+
 		if (!prev->mm) {                        // from kernel
 			/* will mmdrop() in finish_task_switch(). */
 			rq->prev_mm = prev->active_mm;
@@ -4043,6 +4079,15 @@ context_switch(struct rq *rq, struct task_struct *prev,
 	switch_to(prev, next, prev);
 	barrier();
 
+	/*
+	 * If 'next' is on its way to the oob stage, don't run the
+	 * context switch epilogue just yet. We will do that at some
+	 * point later, when the task switches back to the in-band
+	 * stage.
+	 */
+	if (unlikely(inband_switch_tail()))
+		return NULL;
+
 	return finish_task_switch(prev);
 }
 
@@ -4557,6 +4602,8 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt)
 		panic("corrupted shadow stack detected inside scheduler\n");
 #endif
 
+	check_inband_stage();
+
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
 	if (!preempt && prev->state && prev->non_block_count) {
 		printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n",
@@ -4682,7 +4729,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
  *
  * WARNING: must be called with preemption disabled!
  */
-static void __sched notrace __schedule(bool preempt)
+static int __sched notrace __schedule(bool preempt)
 {
 	struct task_struct *prev, *next;
 	unsigned long *switch_count;
@@ -4802,12 +4849,17 @@ static void __sched notrace __schedule(bool preempt)
 
 		/* Also unlocks the rq: */
 		rq = context_switch(rq, prev, next, &rf);
+		if (dovetailing() && rq == NULL)
+			/* Task moved to the oob stage. */
+			return 1;
 	} else {
 		rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
 		rq_unlock_irq(rq, &rf);
 	}
 
 	balance_callback(rq);
+
+	return 0;
 }
 
 void __noreturn do_task_dead(void)
@@ -4879,7 +4931,8 @@ asmlinkage __visible void __sched schedule(void)
 	sched_submit_work(tsk);
 	do {
 		preempt_disable();
-		__schedule(false);
+		if (__schedule(false))
+			return;
 		sched_preempt_enable_no_resched();
 	} while (need_resched());
 	sched_update_worker(tsk);
@@ -4960,7 +5013,8 @@ static void __sched notrace preempt_schedule_common(void)
 		 */
 		preempt_disable_notrace();
 		preempt_latency_start(1);
-		__schedule(true);
+		if (__schedule(true))
+			return;
 		preempt_latency_stop(1);
 		preempt_enable_no_resched_notrace();
 
@@ -4982,7 +5036,7 @@ asmlinkage __visible void __sched notrace preempt_schedule(void)
 	 * If there is a non-zero preempt_count or interrupts are disabled,
 	 * we do not want to preempt the current task. Just return..
 	 */
-	if (likely(!preemptible()))
+	if (likely(!running_inband() || !preemptible()))
 		return;
 
 	preempt_schedule_common();
@@ -5008,7 +5062,7 @@ asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
 {
 	enum ctx_state prev_ctx;
 
-	if (likely(!preemptible()))
+	if (likely(!running_inband() || !preemptible()))
 		return;
 
 	do {
@@ -5049,23 +5103,41 @@ EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
  * off of irq context.
  * Note, that this is called and return with irqs disabled. This will
  * protect us against recursive calling from irq.
+ *
+ * IRQ pipeline: we are called with hard irqs off, synchronize the
+ * pipeline then return the same way, so that the in-band log is
+ * guaranteed empty and further interrupt delivery is postponed by the
+ * hardware until have exited the kernel.
  */
 asmlinkage __visible void __sched preempt_schedule_irq(void)
 {
 	enum ctx_state prev_state;
 
+	if (irq_pipeline_debug()) {
+		/* Catch any weirdness in pipelined entry code. */
+		if (WARN_ON_ONCE(!running_inband()))
+			return;
+		WARN_ON_ONCE(!hard_irqs_disabled());
+	}
+
+	hard_cond_local_irq_enable();
+
 	/* Catch callers which need to be fixed */
 	BUG_ON(preempt_count() || !irqs_disabled());
 
 	prev_state = exception_enter();
 
-	do {
+	for (;;) {
 		preempt_disable();
 		local_irq_enable();
 		__schedule(true);
+		sync_inband_irqs();
 		local_irq_disable();
 		sched_preempt_enable_no_resched();
-	} while (need_resched());
+		if (!need_resched())
+			break;
+		hard_cond_local_irq_enable();
+	}
 
 	exception_exit(prev_state);
 }
@@ -8892,6 +8964,233 @@ struct cgroup_subsys cpu_cgrp_subsys = {
 
 #endif	/* CONFIG_CGROUP_SCHED */
 
+#ifdef CONFIG_DOVETAIL
+
+int dovetail_leave_inband(void)
+{
+	struct task_struct *p = current;
+	struct irq_pipeline_data *pd;
+	unsigned long flags;
+
+	preempt_disable();
+
+	pd = raw_cpu_ptr(&irq_pipeline);
+
+	if (WARN_ON_ONCE(dovetail_debug() && pd->task_inflight))
+		goto out;	/* Paranoid. */
+
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+	pd->task_inflight = p;
+	/*
+	 * The scope of the off-stage state is broader than _TLF_OOB,
+	 * in that it includes the transition path from the in-band
+	 * context to the oob stage.
+	 */
+	set_thread_local_flags(_TLF_OFFSTAGE);
+	set_current_state(TASK_INTERRUPTIBLE);
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+	sched_submit_work(p);
+	/*
+	 * The current task is scheduled out from the inband stage,
+	 * before resuming on the oob stage. Since this code stands
+	 * for the scheduling tail of the oob scheduler,
+	 * arch_dovetail_switch_finish() is called to perform
+	 * architecture-specific fixups (e.g. fpu context reload).
+	 */
+	if (likely(__schedule(false))) {
+		arch_dovetail_switch_finish(false);
+		return 0;
+	}
+
+	clear_thread_local_flags(_TLF_OFFSTAGE);
+	pd->task_inflight = NULL;
+out:
+	preempt_enable();
+
+	return -ERESTARTSYS;
+}
+EXPORT_SYMBOL_GPL(dovetail_leave_inband);
+
+void dovetail_resume_inband(void)
+{
+	struct task_struct *p;
+	struct rq *rq;
+
+	p = __this_cpu_read(irq_pipeline.rqlock_owner);
+	if (WARN_ON_ONCE(dovetail_debug() && p == NULL))
+		return;
+
+	if (WARN_ON_ONCE(dovetail_debug() && (preempt_count() & STAGE_MASK)))
+		return;
+
+	rq = finish_task_switch(p);
+	balance_callback(rq);
+	preempt_enable();
+	oob_trampoline();
+}
+EXPORT_SYMBOL_GPL(dovetail_resume_inband);
+
+#ifdef CONFIG_KVM
+
+#include <linux/kvm_host.h>
+
+static inline void notify_guest_preempt(void)
+{
+	struct kvm_oob_notifier *nfy;
+	struct irq_pipeline_data *p;
+
+	p = raw_cpu_ptr(&irq_pipeline);
+	nfy = p->vcpu_notify;
+	if (unlikely(nfy))
+		nfy->handler(nfy);
+}
+#else
+static inline void notify_guest_preempt(void)
+{ }
+#endif
+
+bool dovetail_context_switch(struct dovetail_altsched_context *out,
+			struct dovetail_altsched_context *in,
+			bool leave_inband)
+{
+	unsigned long pc __maybe_unused, lockdep_irqs;
+	struct task_struct *next, *prev, *last;
+	struct mm_struct *prev_mm, *next_mm;
+	bool inband_tail = false;
+
+	WARN_ON_ONCE(dovetail_debug() && on_pipeline_entry());
+
+	if (leave_inband) {
+		struct task_struct *tsk = current;
+		/*
+		 * We are about to leave the current inband context
+		 * for switching to an out-of-band task, save the
+		 * preempted context information.
+		 */
+		out->task = tsk;
+		out->active_mm = tsk->active_mm;
+		/*
+		 * Switching out-of-band may require some housekeeping
+		 * from a kernel VM which might currently run guest
+		 * code, notify it about the upcoming preemption.
+		 */
+		notify_guest_preempt();
+	}
+
+	arch_dovetail_switch_prepare(leave_inband);
+
+	next = in->task;
+	prev = out->task;
+	prev_mm = out->active_mm;
+	next_mm = in->active_mm;
+
+	if (next_mm == NULL) {
+		in->active_mm = prev_mm;
+		in->borrowed_mm = true;
+		enter_lazy_tlb(prev_mm, next);
+	} else {
+		switch_oob_mm(prev_mm, next_mm, next);
+		/*
+		 * We might be switching back to the inband context
+		 * which we preempted earlier, shortly after "current"
+		 * dropped its mm context in the do_exit() path
+		 * (next->mm == NULL). In such a case, a lazy TLB
+		 * state is expected when leaving the mm.
+		 */
+		if (next->mm == NULL)
+			enter_lazy_tlb(prev_mm, next);
+	}
+
+	if (out->borrowed_mm) {
+		out->borrowed_mm = false;
+		out->active_mm = NULL;
+	}
+
+	/*
+	 * Tasks running out-of-band may alter the (in-band)
+	 * preemption count as long as they don't trigger an in-band
+	 * rescheduling, which Dovetail properly blocks.
+	 *
+	 * If the preemption count is not stack-based but a global
+	 * per-cpu variable instead, changing it has a globally
+	 * visible side-effect though, which is a problem if the
+	 * out-of-band task is preempted and schedules away before the
+	 * change is rolled back: this may cause the in-band context
+	 * to later resume with a broken preemption count.
+	 *
+	 * For this reason, the preemption count of any context which
+	 * blocks from the out-of-band stage is carried over and
+	 * restored across switches, emulating a stack-based
+	 * storage.
+	 *
+	 * Eventually, the count is reset to FORK_PREEMPT_COUNT upon
+	 * transition from out-of-band to in-band stage, reinstating
+	 * the value in effect when the converse transition happened
+	 * at some point before.
+	 */
+	if (IS_ENABLED(CONFIG_HAVE_PERCPU_PREEMPT_COUNT))
+		pc = preempt_count();
+
+	/*
+	 * Like the preemption count and for the same reason, the irq
+	 * state maintained by lockdep must be preserved across
+	 * switches.
+	 */
+	lockdep_irqs = lockdep_read_irqs_state();
+
+	switch_to(prev, next, last);
+	barrier();
+
+	if (check_hard_irqs_disabled())
+		hard_local_irq_disable();
+
+	/*
+	 * If we entered this routine for switching to an out-of-band
+	 * task but don't have _TLF_OOB set for the current context
+	 * when resuming, this portion of code is the switch tail of
+	 * the inband schedule() routine, finalizing a transition to
+	 * the inband stage for the current task. Update the stage
+	 * level as/if required.
+	 */
+	if (unlikely(!leave_inband && !test_thread_local_flags(_TLF_OOB))) {
+		if (IS_ENABLED(CONFIG_HAVE_PERCPU_PREEMPT_COUNT))
+			preempt_count_set(FORK_PREEMPT_COUNT);
+		else if (unlikely(dovetail_debug() &&
+					!(preempt_count() & STAGE_MASK)))
+			WARN_ON_ONCE(1);
+		else
+			preempt_count_sub(STAGE_OFFSET);
+
+		lockdep_write_irqs_state(lockdep_irqs);
+
+		/*
+		 * Fixup the interrupt state conversely to what
+		 * inband_switch_tail() does for the opposite stage
+		 * switching direction.
+		 */
+		stall_inband();
+		trace_hardirqs_off();
+		inband_tail = true;
+	} else {
+		if (IS_ENABLED(CONFIG_HAVE_PERCPU_PREEMPT_COUNT))
+			preempt_count_set(pc);
+
+		lockdep_write_irqs_state(lockdep_irqs);
+	}
+
+	arch_dovetail_switch_finish(leave_inband);
+
+	/*
+	 * inband_tail is true whenever we are finalizing a transition
+	 * to the inband stage from the oob context for current. See
+	 * above.
+	 */
+	return inband_tail;
+}
+EXPORT_SYMBOL_GPL(dovetail_context_switch);
+
+#endif /* CONFIG_DOVETAIL */
+
 void dump_cpu_task(int cpu)
 {
 	pr_info("Task dump for CPU %d:\n", cpu);
diff --git a/kernel/kernel/sched/idle.c b/kernel/kernel/sched/idle.c
index 6dc7d9ae52..cc2710e625 100644
--- a/kernel/kernel/sched/idle.c
+++ b/kernel/kernel/sched/idle.c
@@ -80,6 +80,7 @@ void __weak arch_cpu_idle_dead(void) { }
 void __weak arch_cpu_idle(void)
 {
 	cpu_idle_force_poll = 1;
+	hard_local_irq_enable();
 	raw_local_irq_enable();
 }
 
@@ -87,13 +88,18 @@ void __weak arch_cpu_idle(void)
  * default_idle_call - Default CPU idle routine.
  *
  * To use when the cpuidle framework cannot be used.
+ *
+ * When interrupts are pipelined, this call is entered with hard irqs
+ * on and the in-band stage is stalled. Returns with hard irqs on,
+ * in-band stage stalled. irq_cpuidle_enter() then turns off hard irqs
+ * before synchronizing irqs, making sure we have no event lingering
+ * in the interrupt log as we go for a nap.
  */
 void __cpuidle default_idle_call(void)
 {
 	if (current_clr_polling_and_test()) {
-		local_irq_enable();
-	} else {
-
+		local_irq_enable_full();
+	} else if (irq_cpuidle_enter(NULL, NULL)) { /* hard irqs off now */
 		trace_cpu_idle(1, smp_processor_id());
 		stop_critical_timings();
 
@@ -127,6 +133,8 @@ void __cpuidle default_idle_call(void)
 
 		start_critical_timings();
 		trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
+	} else {
+		local_irq_enable_full();
 	}
 }
 
@@ -248,6 +256,13 @@ static void cpuidle_idle_call(void)
 exit_idle:
 	__current_set_polling();
 
+	/*
+	 *  Catch mishandling of the CPU's interrupt disable flag when
+	 *  pipelining IRQs.
+	 */
+	if (WARN_ON_ONCE(irq_pipeline_debug() && hard_irqs_disabled()))
+		hard_local_irq_enable();
+
 	/*
 	 * It is up to the idle functions to reenable local interrupts
 	 */
@@ -300,6 +315,7 @@ static void do_idle(void)
 			cpu_idle_poll();
 		} else {
 			cpuidle_idle_call();
+			WARN_ON_ONCE(irq_pipeline_debug() && hard_irqs_disabled());
 		}
 		arch_cpu_idle_exit();
 	}
diff --git a/kernel/kernel/sched/sched.h b/kernel/kernel/sched/sched.h
index 9e798c5864..2f1c1ab319 100644
--- a/kernel/kernel/sched/sched.h
+++ b/kernel/kernel/sched/sched.h
@@ -52,6 +52,8 @@
 #include <linux/membarrier.h>
 #include <linux/migrate.h>
 #include <linux/mmu_context.h>
+#include <linux/irq_pipeline.h>
+#include <linux/dovetail.h>
 #include <linux/nmi.h>
 #include <linux/proc_fs.h>
 #include <linux/prefetch.h>
diff --git a/kernel/kernel/sched/wait.c b/kernel/kernel/sched/wait.c
index c4f324ad03..c3a42d935a 100644
--- a/kernel/kernel/sched/wait.c
+++ b/kernel/kernel/sched/wait.c
@@ -71,6 +71,8 @@ static int __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode,
 	wait_queue_entry_t *curr, *next;
 	int cnt = 0;
 
+	check_inband_stage();
+
 	lockdep_assert_held(&wq_head->lock);
 
 	if (bookmark && (bookmark->flags & WQ_FLAG_BOOKMARK)) {
diff --git a/kernel/kernel/signal.c b/kernel/kernel/signal.c
index f6ecd01311..afadec27b9 100644
--- a/kernel/kernel/signal.c
+++ b/kernel/kernel/signal.c
@@ -763,6 +763,10 @@ static int dequeue_synchronous_signal(kernel_siginfo_t *info)
 void signal_wake_up_state(struct task_struct *t, unsigned int state)
 {
 	set_tsk_thread_flag(t, TIF_SIGPENDING);
+
+	/* TIF_SIGPENDING must be set prior to notifying. */
+	inband_signal_notify(t);
+
 	/*
 	 * TASK_WAKEKILL also means wake it up in the stopped/traced/killable
 	 * case. We don't check t->state here because there is a race with it
@@ -984,8 +988,11 @@ static inline bool wants_signal(int sig, struct task_struct *p)
 	if (sig == SIGKILL)
 		return true;
 
-	if (task_is_stopped_or_traced(p))
+	if (task_is_stopped_or_traced(p)) {
+		if (!signal_pending(p))
+			inband_signal_notify(p);
 		return false;
+	}
 
 	return task_curr(p) || !task_sigpending(p);
 }
@@ -2145,6 +2152,7 @@ static void ptrace_stop(int exit_code, int why, int clear_code, kernel_siginfo_t
 	 * schedule() will not sleep if there is a pending signal that
 	 * can awaken the task.
 	 */
+	inband_ptstop_notify();
 	set_special_state(TASK_TRACED);
 
 	/*
@@ -2238,6 +2246,8 @@ static void ptrace_stop(int exit_code, int why, int clear_code, kernel_siginfo_t
 		read_unlock(&tasklist_lock);
 	}
 
+	inband_ptcont_notify();
+
 	/*
 	 * We are back.  Now reacquire the siglock before touching
 	 * last_siginfo, so that we are sure to have synchronized with
diff --git a/kernel/kernel/stop_machine.c b/kernel/kernel/stop_machine.c
index c65cfb7f75..49c00e443a 100644
--- a/kernel/kernel/stop_machine.c
+++ b/kernel/kernel/stop_machine.c
@@ -207,8 +207,8 @@ static int multi_cpu_stop(void *data)
 			curstate = newstate;
 			switch (curstate) {
 			case MULTI_STOP_DISABLE_IRQ:
-				local_irq_disable();
 				hard_irq_disable();
+				local_irq_disable();
 				break;
 			case MULTI_STOP_RUN:
 				if (is_active)
@@ -229,6 +229,7 @@ static int multi_cpu_stop(void *data)
 		rcu_momentary_dyntick_idle();
 	} while (curstate != MULTI_STOP_EXIT);
 
+	hard_irq_enable();
 	local_irq_restore(flags);
 	return err;
 }
@@ -629,6 +630,7 @@ int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
 		local_irq_save(flags);
 		hard_irq_disable();
 		ret = (*fn)(data);
+		hard_irq_enable();
 		local_irq_restore(flags);
 
 		return ret;
diff --git a/kernel/kernel/time/Makefile b/kernel/kernel/time/Makefile
index c8f00168af..14cb45c45b 100644
--- a/kernel/kernel/time/Makefile
+++ b/kernel/kernel/time/Makefile
@@ -16,6 +16,7 @@ ifeq ($(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST),y)
 endif
 obj-$(CONFIG_GENERIC_SCHED_CLOCK)		+= sched_clock.o
 obj-$(CONFIG_TICK_ONESHOT)			+= tick-oneshot.o tick-sched.o
+obj-$(CONFIG_IRQ_PIPELINE)			+= tick-proxy.o
 obj-$(CONFIG_HAVE_GENERIC_VDSO)			+= vsyscall.o
 obj-$(CONFIG_DEBUG_FS)				+= timekeeping_debug.o
 obj-$(CONFIG_TEST_UDELAY)			+= test_udelay.o
diff --git a/kernel/kernel/time/clockevents.c b/kernel/kernel/time/clockevents.c
index f5490222e1..da6735d45a 100644
--- a/kernel/kernel/time/clockevents.c
+++ b/kernel/kernel/time/clockevents.c
@@ -97,6 +97,7 @@ static int __clockevents_switch_state(struct clock_event_device *dev,
 	/* Transition with new state-specific callbacks */
 	switch (state) {
 	case CLOCK_EVT_STATE_DETACHED:
+	case CLOCK_EVT_STATE_RESERVED:
 		/* The clockevent device is getting replaced. Shut it down. */
 
 	case CLOCK_EVT_STATE_SHUTDOWN:
@@ -437,6 +438,69 @@ int clockevents_unbind_device(struct clock_event_device *ced, int cpu)
 }
 EXPORT_SYMBOL_GPL(clockevents_unbind_device);
 
+#ifdef CONFIG_IRQ_PIPELINE
+
+/**
+ * clockevents_register_proxy - register a proxy device on the current CPU
+ * @dev:	proxy to register
+ */
+int clockevents_register_proxy(struct clock_proxy_device *dev)
+{
+	struct clock_event_device *proxy_dev, *real_dev;
+	unsigned long flags;
+	u32 freq;
+	int ret;
+
+	raw_spin_lock_irqsave(&clockevents_lock, flags);
+
+	ret = tick_setup_proxy(dev);
+	if (ret)  {
+		raw_spin_unlock_irqrestore(&clockevents_lock, flags);
+		return ret;
+	}
+
+	proxy_dev = &dev->proxy_device;
+	clockevent_set_state(proxy_dev, CLOCK_EVT_STATE_DETACHED);
+
+	list_add(&proxy_dev->list, &clockevent_devices);
+	tick_check_new_device(proxy_dev);
+	clockevents_notify_released();
+
+	raw_spin_unlock_irqrestore(&clockevents_lock, flags);
+
+	real_dev = dev->real_device;
+	freq = (1000000000ULL * real_dev->mult) >> real_dev->shift;
+	printk(KERN_INFO "CPU%d: proxy tick device registered (%u.%02uMHz)\n",
+		 smp_processor_id(), freq / 1000000, (freq / 10000) % 100);
+
+	return ret;
+}
+
+void clockevents_unregister_proxy(struct clock_proxy_device *dev)
+{
+	unsigned long flags;
+	int ret;
+
+	clockevents_register_device(dev->real_device);
+	clockevents_switch_state(dev->real_device, CLOCK_EVT_STATE_DETACHED);
+
+	/*
+	 *  Pop the proxy device, do not give it back to the
+	 *  framework.
+	 */
+	raw_spin_lock_irqsave(&clockevents_lock, flags);
+	ret = clockevents_replace(&dev->proxy_device);
+	raw_spin_unlock_irqrestore(&clockevents_lock, flags);
+
+	if (WARN_ON(ret))
+		return;
+
+	printk(KERN_INFO "CPU%d: proxy tick device unregistered\n",
+		smp_processor_id());
+}
+
+#endif
+
 /**
  * clockevents_register_device - register a clock event device
  * @dev:	device to register
@@ -575,9 +639,18 @@ void clockevents_exchange_device(struct clock_event_device *old,
 	 */
 	if (old) {
 		module_put(old->owner);
-		clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED);
+		/*
+		 * Do not move the device backing a proxy tick device
+		 * to the release list, keep it around but mark it as
+		 * reserved.
+		 */
 		list_del(&old->list);
-		list_add(&old->list, &clockevents_released);
+		if (tick_check_is_proxy(new)) {
+			clockevents_switch_state(old, CLOCK_EVT_STATE_RESERVED);
+		} else {
+			clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED);
+			list_add(&old->list, &clockevents_released);
+		}
 	}
 
 	if (new) {
diff --git a/kernel/kernel/time/clocksource.c b/kernel/kernel/time/clocksource.c
index 74492f0866..2921e18f59 100644
--- a/kernel/kernel/time/clocksource.c
+++ b/kernel/kernel/time/clocksource.c
@@ -1007,8 +1007,8 @@ void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq
 
 	clocksource_update_max_deferment(cs);
 
-	pr_info("%s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n",
-		cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);
+	pr_info("%s: freq: %Lu Hz, mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n",
+		cs->name, (u64)freq * scale, cs->mask, cs->max_cycles, cs->max_idle_ns);
 }
 EXPORT_SYMBOL_GPL(__clocksource_update_freq_scale);
 
@@ -1275,10 +1275,36 @@ static ssize_t available_clocksource_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(available_clocksource);
 
+/**
+ * vdso_clocksource_show - sysfs interface for vDSO type of
+ *      current clocksource
+ * @dev:	unused
+ * @attr:	unused
+ * @buf:	char buffer to be filled with vDSO type
+ */
+static ssize_t vdso_clocksource_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	ssize_t count = 0, type;
+
+	mutex_lock(&clocksource_mutex);
+	type = curr_clocksource->vdso_type;
+	count = snprintf(buf, PAGE_SIZE, "%s\n",
+			type == CLOCKSOURCE_VDSO_NONE ?	"none" :
+			type == CLOCKSOURCE_VDSO_ARCHITECTED ?	"architected" :
+			"mmio");
+	mutex_unlock(&clocksource_mutex);
+
+	return count;
+}
+static DEVICE_ATTR_RO(vdso_clocksource);
+
 static struct attribute *clocksource_attrs[] = {
 	&dev_attr_current_clocksource.attr,
 	&dev_attr_unbind_clocksource.attr,
 	&dev_attr_available_clocksource.attr,
+	&dev_attr_vdso_clocksource.attr,
 	NULL
 };
 ATTRIBUTE_GROUPS(clocksource);
diff --git a/kernel/kernel/time/hrtimer.c b/kernel/kernel/time/hrtimer.c
index 544ce87ba3..e52623cedf 100644
--- a/kernel/kernel/time/hrtimer.c
+++ b/kernel/kernel/time/hrtimer.c
@@ -873,6 +873,7 @@ void clock_was_set(void)
 	on_each_cpu(retrigger_next_event, NULL, 1);
 #endif
 	timerfd_clock_was_set();
+	inband_clock_was_set();
 }
 
 static void clock_was_set_work(struct work_struct *work)
diff --git a/kernel/kernel/time/tick-broadcast.c b/kernel/kernel/time/tick-broadcast.c
index 086d36b240..e3d15b39b0 100644
--- a/kernel/kernel/time/tick-broadcast.c
+++ b/kernel/kernel/time/tick-broadcast.c
@@ -796,6 +796,23 @@ static int ___tick_broadcast_oneshot_control(enum tick_broadcast_state state,
 	int ret = 0;
 	ktime_t now;
 
+	/*
+	 * If there is no broadcast device, tell the caller not to go
+	 * into deep idle.
+	 */
+	if (!tick_broadcast_device.evtdev)
+		return -EBUSY;
+
+	dev = this_cpu_ptr(&tick_cpu_device)->evtdev;
+
+	/*
+	 * If proxying the hardware timer for high-precision tick
+	 * delivery to the out-of-band stage, the whole broadcast
+	 * dance is a no-go. Deny entering deep idle.
+	 */
+	if (dev->features & CLOCK_EVT_FEAT_PROXY)
+		return -EBUSY;
+
 	raw_spin_lock(&tick_broadcast_lock);
 	bc = tick_broadcast_device.evtdev;
 
diff --git a/kernel/kernel/time/tick-common.c b/kernel/kernel/time/tick-common.c
index 572b4c032e..3f4b7ecd86 100644
--- a/kernel/kernel/time/tick-common.c
+++ b/kernel/kernel/time/tick-common.c
@@ -248,7 +248,8 @@ static void tick_setup_device(struct tick_device *td,
 	} else {
 		handler = td->evtdev->event_handler;
 		next_event = td->evtdev->next_event;
-		td->evtdev->event_handler = clockevents_handle_noop;
+		if (!clockevent_state_reserved(td->evtdev))
+			td->evtdev->event_handler = clockevents_handle_noop;
 	}
 
 	td->evtdev = newdev;
@@ -330,6 +331,12 @@ static bool tick_check_preferred(struct clock_event_device *curdev,
 bool tick_check_replacement(struct clock_event_device *curdev,
 			    struct clock_event_device *newdev)
 {
+	/*
+	 * Never replace an active proxy except when unregistering it.
+	 */
+	if (tick_check_is_proxy(curdev))
+		return false;
+
 	if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
 		return false;
 
@@ -350,6 +357,9 @@ void tick_check_new_device(struct clock_event_device *newdev)
 	td = &per_cpu(tick_cpu_device, cpu);
 	curdev = td->evtdev;
 
+	if (tick_check_is_proxy(curdev))
+		goto out_bc;
+
 	/* cpu local device ? */
 	if (!tick_check_percpu(curdev, newdev, cpu))
 		goto out_bc;
@@ -367,7 +377,12 @@ void tick_check_new_device(struct clock_event_device *newdev)
 	 * not give it back to the clockevents layer !
 	 */
 	if (tick_is_broadcast_device(curdev)) {
-		clockevents_shutdown(curdev);
+		if (tick_check_is_proxy(newdev)) {
+			list_del(&curdev->list);
+			clockevents_switch_state(curdev, CLOCK_EVT_STATE_RESERVED);
+		} else {
+			clockevents_shutdown(curdev);
+		}
 		curdev = NULL;
 	}
 	clockevents_exchange_device(curdev, newdev);
diff --git a/kernel/kernel/time/tick-internal.h b/kernel/kernel/time/tick-internal.h
index ab9cb68b87..1fc4bfc454 100644
--- a/kernel/kernel/time/tick-internal.h
+++ b/kernel/kernel/time/tick-internal.h
@@ -48,15 +48,26 @@ static inline void clockevent_set_state(struct clock_event_device *dev,
 	dev->state_use_accessors = state;
 }
 
+static inline bool tick_check_is_proxy(struct clock_event_device *dev)
+{
+	if (!irqs_pipelined())
+		return false;
+
+	return dev && dev->features & CLOCK_EVT_FEAT_PROXY;
+}
+
 extern void clockevents_shutdown(struct clock_event_device *dev);
 extern void clockevents_exchange_device(struct clock_event_device *old,
 					struct clock_event_device *new);
-extern void clockevents_switch_state(struct clock_event_device *dev,
-				     enum clock_event_state state);
 extern int clockevents_program_event(struct clock_event_device *dev,
 				     ktime_t expires, bool force);
 extern void clockevents_handle_noop(struct clock_event_device *dev);
 extern int __clockevents_update_freq(struct clock_event_device *dev, u32 freq);
+#ifdef CONFIG_IRQ_PIPELINE
+int clockevents_register_proxy(struct clock_proxy_device *dev);
+extern void clockevents_unregister_proxy(struct clock_proxy_device *dev);
+int tick_setup_proxy(struct clock_proxy_device *dev);
+#endif
 extern ssize_t sysfs_get_uname(const char *buf, char *dst, size_t cnt);
 
 /* Broadcasting support */
diff --git a/kernel/kernel/time/tick-proxy.c b/kernel/kernel/time/tick-proxy.c
new file mode 100644
index 0000000000..5a877987a8
--- /dev/null
+++ b/kernel/kernel/time/tick-proxy.c
@@ -0,0 +1,466 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2017 Philippe Gerum  <rpm@xenomai.org>.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+#include <linux/delay.h>
+#include <linux/smp.h>
+#include <linux/err.h>
+#include <linux/cpumask.h>
+#include <linux/clockchips.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irq_pipeline.h>
+#include <linux/stop_machine.h>
+#include <linux/slab.h>
+#include "tick-internal.h"
+
+static unsigned int proxy_tick_irq;
+
+static DEFINE_MUTEX(proxy_mutex);
+
+static DEFINE_PER_CPU(struct clock_proxy_device, proxy_tick_device);
+
+static inline struct clock_event_device *
+get_real_tick_device(struct clock_event_device *proxy_dev)
+{
+	return container_of(proxy_dev, struct clock_proxy_device, proxy_device)->real_device;
+}
+
+static void proxy_event_handler(struct clock_event_device *real_dev)
+{
+	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
+	struct clock_event_device *proxy_dev = &dev->proxy_device;
+
+	proxy_dev->event_handler(proxy_dev);
+}
+
+static int proxy_set_state_oneshot(struct clock_event_device *dev)
+{
+	struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+	int ret;
+
+	flags = hard_local_irq_save();
+	ret = real_dev->set_state_oneshot(real_dev);
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+static int proxy_set_state_periodic(struct clock_event_device *dev)
+{
+	struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+	int ret;
+
+	flags = hard_local_irq_save();
+	ret = real_dev->set_state_periodic(real_dev);
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+static int proxy_set_state_oneshot_stopped(struct clock_event_device *dev)
+{
+        struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+	int ret;
+
+	flags = hard_local_irq_save();
+	ret = real_dev->set_state_oneshot_stopped(real_dev);
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+static int proxy_set_state_shutdown(struct clock_event_device *dev)
+{
+        struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+	int ret;
+
+	flags = hard_local_irq_save();
+	ret = real_dev->set_state_shutdown(real_dev);
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+static void proxy_suspend(struct clock_event_device *dev)
+{
+        struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	real_dev->suspend(real_dev);
+	hard_local_irq_restore(flags);
+}
+
+static void proxy_resume(struct clock_event_device *dev)
+{
+        struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	real_dev->resume(real_dev);
+	hard_local_irq_restore(flags);
+}
+
+static int proxy_tick_resume(struct clock_event_device *dev)
+{
+        struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+	int ret;
+
+	flags = hard_local_irq_save();
+	ret = real_dev->tick_resume(real_dev);
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+static void proxy_broadcast(const struct cpumask *mask)
+{
+	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
+        struct clock_event_device *real_dev = dev->real_device;
+	unsigned long flags;
+
+	flags = hard_local_irq_save();
+	real_dev->broadcast(mask);
+	hard_local_irq_restore(flags);
+}
+
+static int proxy_set_next_event(unsigned long delay,
+				struct clock_event_device *dev)
+{
+	struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+	int ret;
+
+	flags = hard_local_irq_save();
+	ret = real_dev->set_next_event(delay, real_dev);
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+static int proxy_set_next_ktime(ktime_t expires,
+				struct clock_event_device *dev)
+{
+	struct clock_event_device *real_dev = get_real_tick_device(dev);
+	unsigned long flags;
+	int ret;
+
+	flags = hard_local_irq_save();
+	ret = real_dev->set_next_ktime(expires, real_dev);
+	hard_local_irq_restore(flags);
+
+	return ret;
+}
+
+static irqreturn_t proxy_irq_handler(int sirq, void *dev_id)
+{
+	struct clock_event_device *evt;
+
+	/*
+	 * Tricky: we may end up running this in-band IRQ handler
+	 * because tick_notify_proxy() was posted either:
+	 *
+	 * - from the out-of-band stage via ->handle_oob_event() for
+	 * emulating an in-band tick.  In this case, the active tick
+	 * device for the in-band timing core is the proxy device,
+	 * whose event handler is still the same than the real tick
+	 * device's.
+	 *
+	 * - directly by the clock chip driver on the local CPU via
+	 * clockevents_handle_event(), for propagating a tick to the
+	 * in-band stage nobody from the out-of-band stage is
+	 * interested on i.e. no proxy device was registered on the
+	 * receiving CPU, which was excluded from @cpumask in the call
+	 * to tick_install_proxy(). In this case, the active tick
+	 * device for the in-band timing core is a real clock event
+	 * device.
+	 *
+	 * In both cases, we are running on the in-band stage, and we
+	 * should fire the event handler of the currently active tick
+	 * device for the in-band timing core.
+	 */
+	evt = raw_cpu_ptr(&tick_cpu_device)->evtdev;
+	evt->event_handler(evt);
+
+	return IRQ_HANDLED;
+}
+
+#define interpose_proxy_handler(__proxy, __real, __h)		\
+	do {							\
+		if ((__real)->__h)				\
+			(__proxy)->__h = proxy_ ## __h;		\
+	} while (0)
+
+/*
+ * Setup a proxy which is about to override the tick device on the
+ * current CPU. Called with clockevents_lock held and irqs off so that
+ * the tick device does not change under our feet.
+ */
+int tick_setup_proxy(struct clock_proxy_device *dev)
+{
+	struct clock_event_device *proxy_dev, *real_dev;
+
+	real_dev = raw_cpu_ptr(&tick_cpu_device)->evtdev;
+	if ((real_dev->features &
+			(CLOCK_EVT_FEAT_PIPELINE|CLOCK_EVT_FEAT_ONESHOT))
+		!= (CLOCK_EVT_FEAT_PIPELINE|CLOCK_EVT_FEAT_ONESHOT)) {
+		WARN(1, "cannot use clockevent device %s in proxy mode!",
+			real_dev->name);
+		return -ENODEV;
+	}
+
+ 	/*
+ 	 * The assumption is that neither us nor clockevents_register_proxy()
+	 * can fail afterwards, so this is ok to advertise the new proxy as
+	 * built by setting dev->real_device early.
+ 	 */
+	dev->real_device = real_dev;
+	dev->__original_handler = real_dev->event_handler;
+
+	/*
+	 * Inherit the feature bits since the proxy device has the
+	 * same capabilities than the real one we are overriding
+	 * (including CLOCK_EVT_FEAT_C3STOP if present).
+	 */
+	proxy_dev = &dev->proxy_device;
+	memset(proxy_dev, 0, sizeof(*proxy_dev));
+	proxy_dev->features = real_dev->features |
+		CLOCK_EVT_FEAT_PERCPU | CLOCK_EVT_FEAT_PROXY;
+	proxy_dev->name = "proxy";
+	proxy_dev->irq = real_dev->irq;
+	proxy_dev->bound_on = -1;
+	proxy_dev->cpumask = cpumask_of(smp_processor_id());
+	proxy_dev->rating = real_dev->rating + 1;
+	proxy_dev->mult = real_dev->mult;
+	proxy_dev->shift = real_dev->shift;
+	proxy_dev->max_delta_ticks = real_dev->max_delta_ticks;
+	proxy_dev->min_delta_ticks = real_dev->min_delta_ticks;
+	proxy_dev->max_delta_ns = real_dev->max_delta_ns;
+	proxy_dev->min_delta_ns = real_dev->min_delta_ns;
+	/*
+	 * Interpose default handlers which are safe wrt preemption by
+	 * the out-of-band stage.
+	 */
+	interpose_proxy_handler(proxy_dev, real_dev, set_state_oneshot);
+	interpose_proxy_handler(proxy_dev, real_dev, set_state_oneshot_stopped);
+	interpose_proxy_handler(proxy_dev, real_dev, set_state_periodic);
+	interpose_proxy_handler(proxy_dev, real_dev, set_state_shutdown);
+	interpose_proxy_handler(proxy_dev, real_dev, suspend);
+	interpose_proxy_handler(proxy_dev, real_dev, resume);
+	interpose_proxy_handler(proxy_dev, real_dev, tick_resume);
+	interpose_proxy_handler(proxy_dev, real_dev, broadcast);
+	interpose_proxy_handler(proxy_dev, real_dev, set_next_event);
+	interpose_proxy_handler(proxy_dev, real_dev, set_next_ktime);
+
+	dev->__setup_handler(dev);
+
+	return 0;
+}
+
+static int enable_oob_timer(void *arg) /* hard_irqs_disabled() */
+{
+	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
+	struct clock_event_device *real_dev;
+
+	/*
+	 * Install the out-of-band handler on this CPU's real clock
+	 * device, then turn on out-of-band mode for the associated
+	 * IRQ (duplicates are silently ignored if the IRQ is common
+	 * to multiple CPUs).
+	 */
+	real_dev = dev->real_device;
+	real_dev->event_handler = dev->handle_oob_event;
+	real_dev->features |= CLOCK_EVT_FEAT_OOB;
+	barrier();
+
+	/*
+	 * irq_switch_oob() grabs the IRQ descriptor lock which is
+	 * hybrid, so that is fine to invoke this routine with hard
+	 * IRQs off.
+	 */
+	irq_switch_oob(real_dev->irq, true);
+
+	return 0;
+}
+
+struct proxy_install_arg {
+	void (*setup_proxy)(struct clock_proxy_device *dev);
+	int result;
+};
+
+static void register_proxy_device(void *arg) /* irqs_disabled() */
+{
+	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
+	struct proxy_install_arg *req = arg;
+	int ret;
+
+	dev->__setup_handler = req->setup_proxy;
+	ret = clockevents_register_proxy(dev);
+	if (ret) {
+		if (!req->result)
+			req->result = ret;
+	} else {
+		dev->real_device->event_handler = proxy_event_handler;
+	}
+}
+
+int tick_install_proxy(void (*setup_proxy)(struct clock_proxy_device *dev),
+		const struct cpumask *cpumask)
+{
+	struct proxy_install_arg arg;
+	int ret, sirq;
+
+	mutex_lock(&proxy_mutex);
+
+	ret = -EAGAIN;
+	if (proxy_tick_irq)
+		goto out;
+
+	sirq = irq_create_direct_mapping(synthetic_irq_domain);
+	if (WARN_ON(sirq == 0))
+		goto out;
+
+	ret = __request_percpu_irq(sirq, proxy_irq_handler,
+				   IRQF_NO_THREAD, /* no IRQF_TIMER here. */
+				   "proxy tick",
+				   &proxy_tick_device);
+	if (WARN_ON(ret)) {
+		irq_dispose_mapping(sirq);
+		goto out;
+	}
+
+	proxy_tick_irq = sirq;
+	barrier();
+
+	/*
+	 * Install a proxy tick device on each CPU. As the proxy
+	 * device is picked, the previous (real) tick device is
+	 * switched to reserved state by the clockevent core.
+	 * Immediately after, the proxy device starts controlling the
+	 * real device under the hood to carry out the timing requests
+	 * it receives.
+	 *
+	 * For a short period of time, after the proxy device is
+	 * installed, and until the real device IRQ is switched to
+	 * out-of-band mode, the flow is as follows:
+	 *
+	 *    [inband timing request]
+	 *        proxy_dev->set_next_event(proxy_dev)
+	 *            oob_program_event(proxy_dev)
+	 *                real_dev->set_next_event(real_dev)
+	 *        ...
+	 *        <tick event>
+	 *        original_timer_handler() [in-band stage]
+	 *            clockevents_handle_event(real_dev)
+	 *               proxy_event_handler(real_dev)
+	 *                  inband_event_handler(proxy_dev)
+	 *
+	 * Eventually, we substitute the original (in-band) clock
+	 * event handler with the out-of-band handler for the real
+	 * clock event device, then turn on out-of-band mode for the
+	 * timer IRQ associated to the latter. These two steps are
+	 * performed over a stop_machine() context, so that no tick
+	 * can race with this code while we swap handlers.
+	 *
+	 * Once the hand over is complete, the flow is as follows:
+	 *
+	 *    [inband timing request]
+	 *        proxy_dev->set_next_event(proxy_dev)
+	 *            oob_program_event(proxy_dev)
+	 *                real_dev->set_next_event(real_dev)
+	 *        ...
+	 *        <tick event>
+	 *        inband_event_handler() [out-of-band stage]
+	 *            clockevents_handle_event(real_dev)
+	 *                handle_oob_event(proxy_dev)
+	 *                    ...(inband tick emulation)...
+	 *                         tick_notify_proxy()
+	 *        ...
+	 *        proxy_irq_handler(proxy_dev) [in-band stage]
+	 *            clockevents_handle_event(proxy_dev)
+	 *                inband_event_handler(proxy_dev)
+	 */
+	arg.setup_proxy = setup_proxy;
+	arg.result = 0;
+	on_each_cpu_mask(cpumask, register_proxy_device, &arg, true);
+	if (arg.result) {
+		tick_uninstall_proxy(cpumask);
+		return arg.result;
+	}
+
+	/*
+	 * Start ticking from the out-of-band interrupt stage upon
+	 * receipt of out-of-band timer events.
+	 */
+	stop_machine(enable_oob_timer, NULL, cpumask);
+out:
+	mutex_unlock(&proxy_mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(tick_install_proxy);
+
+static int disable_oob_timer(void *arg) /* hard_irqs_disabled() */
+{
+	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
+	struct clock_event_device *real_dev;
+
+	dev = raw_cpu_ptr(&proxy_tick_device);
+	real_dev = dev->real_device;
+	real_dev->event_handler = dev->__original_handler;
+	real_dev->features &= ~CLOCK_EVT_FEAT_OOB;
+	barrier();
+
+	irq_switch_oob(real_dev->irq, false);
+
+	return 0;
+}
+
+static void unregister_proxy_device(void *arg) /* irqs_disabled() */
+{
+	struct clock_proxy_device *dev = raw_cpu_ptr(&proxy_tick_device);
+
+	if (dev->real_device) {
+		clockevents_unregister_proxy(dev);
+		dev->real_device = NULL;
+	}
+}
+
+void tick_uninstall_proxy(const struct cpumask *cpumask)
+{
+	/*
+	 * Undo all we did in tick_install_proxy(), handing over
+	 * control of the tick device back to the inband code.
+	 */
+	mutex_lock(&proxy_mutex);
+	stop_machine(disable_oob_timer, NULL, cpu_online_mask);
+	on_each_cpu_mask(cpumask, unregister_proxy_device, NULL, true);
+	free_percpu_irq(proxy_tick_irq, &proxy_tick_device);
+	irq_dispose_mapping(proxy_tick_irq);
+	proxy_tick_irq = 0;
+	mutex_unlock(&proxy_mutex);
+}
+EXPORT_SYMBOL_GPL(tick_uninstall_proxy);
+
+void tick_notify_proxy(void)
+{
+	/*
+	 * Schedule a tick on the proxy device to occur from the
+	 * in-band stage, which will trigger proxy_irq_handler() at
+	 * some point (i.e. when the in-band stage is back in control
+	 * and not stalled). Note that we might be called from the
+	 * in-band stage in some cases (see proxy_irq_handler()).
+	 */
+	irq_post_inband(proxy_tick_irq);
+}
+EXPORT_SYMBOL_GPL(tick_notify_proxy);
diff --git a/kernel/kernel/time/vsyscall.c b/kernel/kernel/time/vsyscall.c
index 88e6b8ed6c..2b9b786682 100644
--- a/kernel/kernel/time/vsyscall.c
+++ b/kernel/kernel/time/vsyscall.c
@@ -69,16 +69,42 @@ static inline void update_vdso_data(struct vdso_data *vdata,
 	vdso_ts->nsec	= tk->tkr_mono.xtime_nsec;
 }
 
+static void update_generic_mmio(struct vdso_data *vdata, struct timekeeper *tk)
+{
+#ifdef CONFIG_GENERIC_CLOCKSOURCE_VDSO
+	const struct clocksource *cs = tk->tkr_mono.clock;
+	u16 seq;
+
+	if (cs->vdso_type == (vdata->cs_type_seq >> 16))
+		return;
+
+	seq = vdata->cs_type_seq;
+	if (++seq == 0)
+		seq = 1;
+
+	vdata->cs_type_seq = cs->vdso_type << 16 | seq;
+
+	if (cs->vdso_type >= CLOCKSOURCE_VDSO_MMIO)
+		snprintf(vdata->cs_mmdev, sizeof(vdata->cs_mmdev),
+			"/dev/ucs/%u", cs->vdso_type - CLOCKSOURCE_VDSO_MMIO);
+#endif
+}
+
 void update_vsyscall(struct timekeeper *tk)
 {
 	struct vdso_data *vdata = __arch_get_k_vdso_data();
 	struct vdso_timestamp *vdso_ts;
+	unsigned long flags;
 	s32 clock_mode;
 	u64 nsec;
 
+	flags = hard_cond_local_irq_save();
+
 	/* copy vsyscall data */
 	vdso_write_begin(vdata);
 
+	update_generic_mmio(vdata, tk);
+
 	clock_mode = tk->tkr_mono.clock->vdso_clock_mode;
 	vdata[CS_HRES_COARSE].clock_mode	= clock_mode;
 	vdata[CS_RAW].clock_mode		= clock_mode;
@@ -110,13 +136,16 @@ void update_vsyscall(struct timekeeper *tk)
 	 * If the current clocksource is not VDSO capable, then spare the
 	 * update of the high reolution parts.
 	 */
-	if (clock_mode != VDSO_CLOCKMODE_NONE)
+	if (IS_ENABLED(CONFIG_GENERIC_CLOCKSOURCE_VDSO) ||
+	    clock_mode != VDSO_CLOCKMODE_NONE)
 		update_vdso_data(vdata, tk);
 
 	__arch_update_vsyscall(vdata, tk);
 
 	vdso_write_end(vdata);
 
+	hard_cond_local_irq_restore(flags);
+
 	__arch_sync_vdso_data(vdata);
 }
 
diff --git a/kernel/kernel/trace/ftrace.c b/kernel/kernel/trace/ftrace.c
index d97c189695..1e3948987d 100644
--- a/kernel/kernel/trace/ftrace.c
+++ b/kernel/kernel/trace/ftrace.c
@@ -6271,10 +6271,10 @@ static int ftrace_process_locs(struct module *mod,
 	 * reason to cause large interrupt latencies while we do it.
 	 */
 	if (!mod)
-		local_irq_save(flags);
+		flags = hard_local_irq_save();
 	ftrace_update_code(mod, start_pg);
 	if (!mod)
-		local_irq_restore(flags);
+		hard_local_irq_restore(flags);
 	ret = 0;
  out:
 	mutex_unlock(&ftrace_lock);
@@ -6865,9 +6865,9 @@ void __init ftrace_init(void)
 	unsigned long count, flags;
 	int ret;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	ret = ftrace_dyn_arch_init();
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 	if (ret)
 		goto failed;
 
@@ -7022,7 +7022,15 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
 		}
 	} while_for_each_ftrace_op(op);
 out:
-	preempt_enable_notrace();
+	if (irqs_pipelined() && (hard_irqs_disabled() || !running_inband()))
+		/*
+		 * Nothing urgent to schedule here. At latest the
+		 * timer tick will pick up whatever the tracing
+		 * functions kicked off.
+		 */
+		preempt_enable_no_resched_notrace();
+	else
+		preempt_enable_notrace();
 	trace_clear_recursion(bit);
 }
 
diff --git a/kernel/kernel/trace/ring_buffer.c b/kernel/kernel/trace/ring_buffer.c
index 49ebb8c662..391151fcbc 100644
--- a/kernel/kernel/trace/ring_buffer.c
+++ b/kernel/kernel/trace/ring_buffer.c
@@ -3165,8 +3165,8 @@ rb_wakeups(struct trace_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)
 static __always_inline int
 trace_recursive_lock(struct ring_buffer_per_cpu *cpu_buffer)
 {
-	unsigned int val = cpu_buffer->current_context;
-	unsigned long pc = preempt_count();
+	unsigned int val;
+	unsigned long pc = preempt_count(), flags;
 	int bit;
 
 	if (!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
@@ -3175,6 +3175,10 @@ trace_recursive_lock(struct ring_buffer_per_cpu *cpu_buffer)
 		bit = pc & NMI_MASK ? RB_CTX_NMI :
 			pc & HARDIRQ_MASK ? RB_CTX_IRQ : RB_CTX_SOFTIRQ;
 
+	flags = hard_cond_local_irq_save();
+
+	val = cpu_buffer->current_context;
+
 	if (unlikely(val & (1 << (bit + cpu_buffer->nest)))) {
 		/*
 		 * It is possible that this was called by transitioning
@@ -3182,21 +3186,29 @@ trace_recursive_lock(struct ring_buffer_per_cpu *cpu_buffer)
 		 * been updated yet. In this case, use the TRANSITION bit.
 		 */
 		bit = RB_CTX_TRANSITION;
-		if (val & (1 << (bit + cpu_buffer->nest)))
+		if (val & (1 << (bit + cpu_buffer->nest))) {
+			hard_cond_local_irq_restore(flags);
 			return 1;
+		}
 	}
 
 	val |= (1 << (bit + cpu_buffer->nest));
 	cpu_buffer->current_context = val;
 
+	hard_cond_local_irq_restore(flags);
+
 	return 0;
 }
 
 static __always_inline void
 trace_recursive_unlock(struct ring_buffer_per_cpu *cpu_buffer)
 {
+	unsigned long flags;
+
+	flags = hard_cond_local_irq_save();
 	cpu_buffer->current_context &=
 		cpu_buffer->current_context - (1 << cpu_buffer->nest);
+	hard_cond_local_irq_restore(flags);
 }
 
 /* The recursive locking above uses 5 bits */
diff --git a/kernel/kernel/trace/trace.c b/kernel/kernel/trace/trace.c
index 8b1f74eb98..f339a561b1 100644
--- a/kernel/kernel/trace/trace.c
+++ b/kernel/kernel/trace/trace.c
@@ -1129,9 +1129,9 @@ static void tracing_snapshot_instance_cond(struct trace_array *tr,
 		return;
 	}
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	update_max_tr(tr, current, smp_processor_id(), cond_data);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 void tracing_snapshot_instance(struct trace_array *tr)
@@ -1822,7 +1822,7 @@ update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
 	if (tr->stop_count)
 		return;
 
-	WARN_ON_ONCE(!irqs_disabled());
+	WARN_ON_ONCE(!hard_irqs_disabled());
 
 	if (!tr->allocated_snapshot) {
 		/* Only the nop tracer should hit this when disabling */
@@ -1866,7 +1866,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 	if (tr->stop_count)
 		return;
 
-	WARN_ON_ONCE(!irqs_disabled());
+	WARN_ON_ONCE(!hard_irqs_disabled());
 	if (!tr->allocated_snapshot) {
 		/* Only the nop tracer should hit this when disabling */
 		WARN_ON_ONCE(tr->current_trace != &nop_trace);
@@ -2626,12 +2626,14 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned short type,
 	entry->flags =
 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
 		(irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
-#else
+		(hard_irqs_disabled() ? TRACE_FLAG_IRQS_HARDOFF : 0) |
+#elif !defined(CONFIG_IRQ_PIPELINE)
 		TRACE_FLAG_IRQS_NOSUPPORT |
 #endif
 		((pc & NMI_MASK    ) ? TRACE_FLAG_NMI     : 0) |
 		((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
 		((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
+		(running_oob() ? TRACE_FLAG_OOB_STAGE : 0) |
 		(tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
 		(test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
 }
@@ -7085,13 +7087,13 @@ tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
 			ret = tracing_alloc_snapshot_instance(tr);
 		if (ret < 0)
 			break;
-		local_irq_disable();
+		hard_local_irq_disable();
 		/* Now, we're going to swap */
 		if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
 			update_max_tr(tr, current, smp_processor_id(), NULL);
 		else
 			update_max_tr_single(tr, current, iter->cpu_file);
-		local_irq_enable();
+		hard_local_irq_enable();
 		break;
 	default:
 		if (tr->allocated_snapshot) {
diff --git a/kernel/kernel/trace/trace.h b/kernel/kernel/trace/trace.h
index 8d67f7f448..c838a90041 100644
--- a/kernel/kernel/trace/trace.h
+++ b/kernel/kernel/trace/trace.h
@@ -139,11 +139,14 @@ struct kretprobe_trace_entry_head {
 /*
  * trace_flag_type is an enumeration that holds different
  * states when a trace occurs. These are:
- *  IRQS_OFF		- interrupts were disabled
+ *  IRQS_OFF		- interrupts were off (only virtually if pipelining)
  *  IRQS_NOSUPPORT	- arch does not support irqs_disabled_flags
  *  NEED_RESCHED	- reschedule is requested
  *  HARDIRQ		- inside an interrupt handler
  *  SOFTIRQ		- inside a softirq handler
+ *  IRQS_HARDOFF	- interrupts were hard disabled
+ *  OOB_STAGE		- running over the oob stage (assume IRQ tracing
+ *                        support is always available w/ pipelining).
  */
 enum trace_flag_type {
 	TRACE_FLAG_IRQS_OFF		= 0x01,
@@ -153,6 +156,8 @@ enum trace_flag_type {
 	TRACE_FLAG_SOFTIRQ		= 0x10,
 	TRACE_FLAG_PREEMPT_RESCHED	= 0x20,
 	TRACE_FLAG_NMI			= 0x40,
+	TRACE_FLAG_IRQS_HARDOFF		= 0x80,
+	TRACE_FLAG_OOB_STAGE		= TRACE_FLAG_IRQS_NOSUPPORT,
 };
 
 #define TRACE_BUF_SIZE		1024
diff --git a/kernel/kernel/trace/trace_branch.c b/kernel/kernel/trace/trace_branch.c
index eff099123a..e9e754fce1 100644
--- a/kernel/kernel/trace/trace_branch.c
+++ b/kernel/kernel/trace/trace_branch.c
@@ -53,7 +53,7 @@ probe_likely_condition(struct ftrace_likely_data *f, int val, int expect)
 	if (unlikely(!tr))
 		return;
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 	current->trace_recursion |= TRACE_BRANCH_BIT;
 	data = this_cpu_ptr(tr->array_buffer.data);
 	if (atomic_read(&data->disabled))
@@ -87,7 +87,7 @@ probe_likely_condition(struct ftrace_likely_data *f, int val, int expect)
 
  out:
 	current->trace_recursion &= ~TRACE_BRANCH_BIT;
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static inline
diff --git a/kernel/kernel/trace/trace_clock.c b/kernel/kernel/trace/trace_clock.c
index 4702efb00f..79a4cc1d62 100644
--- a/kernel/kernel/trace/trace_clock.c
+++ b/kernel/kernel/trace/trace_clock.c
@@ -97,7 +97,7 @@ u64 notrace trace_clock_global(void)
 	int this_cpu;
 	u64 now, prev_time;
 
-	raw_local_irq_save(flags);
+	flags = hard_local_irq_save();
 
 	this_cpu = raw_smp_processor_id();
 
@@ -139,7 +139,7 @@ u64 notrace trace_clock_global(void)
 		arch_spin_unlock(&trace_clock_struct.lock);
 	}
  out:
-	raw_local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return now;
 }
diff --git a/kernel/kernel/trace/trace_functions.c b/kernel/kernel/trace/trace_functions.c
index 93e20ed642..c4d03382c5 100644
--- a/kernel/kernel/trace/trace_functions.c
+++ b/kernel/kernel/trace/trace_functions.c
@@ -196,7 +196,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
 	 * Need to use raw, since this must be called before the
 	 * recursive protection is performed.
 	 */
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	cpu = raw_smp_processor_id();
 	data = per_cpu_ptr(tr->array_buffer.data, cpu);
 	disabled = atomic_inc_return(&data->disabled);
@@ -208,7 +208,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
 	}
 
 	atomic_dec(&data->disabled);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 static struct tracer_opt func_opts[] = {
diff --git a/kernel/kernel/trace/trace_functions_graph.c b/kernel/kernel/trace/trace_functions_graph.c
index 60d66278aa..8a3d156f2b 100644
--- a/kernel/kernel/trace/trace_functions_graph.c
+++ b/kernel/kernel/trace/trace_functions_graph.c
@@ -169,7 +169,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
 	if (tracing_thresh)
 		return 1;
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	cpu = raw_smp_processor_id();
 	data = per_cpu_ptr(tr->array_buffer.data, cpu);
 	disabled = atomic_inc_return(&data->disabled);
@@ -181,7 +181,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
 	}
 
 	atomic_dec(&data->disabled);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 
 	return ret;
 }
@@ -250,7 +250,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
 		return;
 	}
 
-	local_irq_save(flags);
+	flags = hard_local_irq_save();
 	cpu = raw_smp_processor_id();
 	data = per_cpu_ptr(tr->array_buffer.data, cpu);
 	disabled = atomic_inc_return(&data->disabled);
@@ -259,7 +259,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
 		__trace_graph_return(tr, trace, flags, pc);
 	}
 	atomic_dec(&data->disabled);
-	local_irq_restore(flags);
+	hard_local_irq_restore(flags);
 }
 
 void set_graph_array(struct trace_array *tr)
diff --git a/kernel/kernel/trace/trace_irqsoff.c b/kernel/kernel/trace/trace_irqsoff.c
index ee4571b624..92a816d428 100644
--- a/kernel/kernel/trace/trace_irqsoff.c
+++ b/kernel/kernel/trace/trace_irqsoff.c
@@ -14,6 +14,7 @@
 #include <linux/uaccess.h>
 #include <linux/module.h>
 #include <linux/ftrace.h>
+#include <linux/irqstage.h>
 #include <linux/kprobes.h>
 
 #include "trace.h"
@@ -26,7 +27,7 @@ static int				tracer_enabled __read_mostly;
 
 static DEFINE_PER_CPU(int, tracing_cpu);
 
-static DEFINE_RAW_SPINLOCK(max_trace_lock);
+static DEFINE_HARD_SPINLOCK(max_trace_lock);
 
 enum {
 	TRACER_IRQS_OFF		= (1 << 1),
@@ -44,7 +45,7 @@ static int start_irqsoff_tracer(struct trace_array *tr, int graph);
 static inline int
 preempt_trace(int pc)
 {
-	return ((trace_type & TRACER_PREEMPT_OFF) && pc);
+	return (running_inband() && (trace_type & TRACER_PREEMPT_OFF) && pc);
 }
 #else
 # define preempt_trace(pc) (0)
@@ -55,7 +56,7 @@ static inline int
 irq_trace(void)
 {
 	return ((trace_type & TRACER_IRQS_OFF) &&
-		irqs_disabled());
+		(hard_irqs_disabled() || (running_inband() && irqs_disabled())));
 }
 #else
 # define irq_trace() (0)
@@ -393,7 +394,7 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip, int pc)
 	data->preempt_timestamp = ftrace_now(cpu);
 	data->critical_start = parent_ip ? : ip;
 
-	local_save_flags(flags);
+	stage_save_flags(flags);
 
 	__trace_function(tr, ip, parent_ip, flags, pc);
 
@@ -428,7 +429,7 @@ stop_critical_timing(unsigned long ip, unsigned long parent_ip, int pc)
 
 	atomic_inc(&data->disabled);
 
-	local_save_flags(flags);
+	stage_save_flags(flags);
 	__trace_function(tr, ip, parent_ip, flags, pc);
 	check_critical_timing(tr, data, parent_ip ? : ip, cpu);
 	data->critical_start = 0;
diff --git a/kernel/kernel/trace/trace_output.c b/kernel/kernel/trace/trace_output.c
index 000e9dc224..385893474f 100644
--- a/kernel/kernel/trace/trace_output.c
+++ b/kernel/kernel/trace/trace_output.c
@@ -445,14 +445,19 @@ int trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
 	int hardirq;
 	int softirq;
 	int nmi;
+	int oob;
 
 	nmi = entry->flags & TRACE_FLAG_NMI;
 	hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
 	softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
+	oob = irqs_pipelined() && (entry->flags & TRACE_FLAG_OOB_STAGE);
 
 	irqs_off =
+		(entry->flags & (TRACE_FLAG_IRQS_OFF|TRACE_FLAG_IRQS_HARDOFF)) ==
+		(TRACE_FLAG_IRQS_OFF|TRACE_FLAG_IRQS_HARDOFF) ? '*' :
+		(entry->flags & TRACE_FLAG_IRQS_HARDOFF) ? 'D' :
 		(entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
-		(entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' :
+		!irqs_pipelined() && (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' :
 		'.';
 
 	switch (entry->flags & (TRACE_FLAG_NEED_RESCHED |
@@ -472,6 +477,8 @@ int trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
 	}
 
 	hardsoft_irq =
+		(nmi && oob)  ? '#' :
+		oob           ? '~' :
 		(nmi && hardirq)     ? 'Z' :
 		nmi                  ? 'z' :
 		(hardirq && softirq) ? 'H' :
diff --git a/kernel/kernel/trace/trace_preemptirq.c b/kernel/kernel/trace/trace_preemptirq.c
index 4593f16da9..42b1790fe6 100644
--- a/kernel/kernel/trace/trace_preemptirq.c
+++ b/kernel/kernel/trace/trace_preemptirq.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <linux/ftrace.h>
 #include <linux/kprobes.h>
+#include <linux/irq_pipeline.h>
 #include "trace.h"
 
 #define CREATE_TRACE_POINTS
@@ -133,6 +134,57 @@ __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
 }
 EXPORT_SYMBOL(trace_hardirqs_off_caller);
 NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
+
+#ifdef CONFIG_IRQ_PIPELINE
+
+void trace_hardirqs_off_pipelined(void)
+{
+	WARN_ON(irq_pipeline_debug() && !hard_irqs_disabled());
+
+	if (running_inband())
+		trace_hardirqs_off();
+}
+EXPORT_SYMBOL(trace_hardirqs_off_pipelined);
+NOKPROBE_SYMBOL(trace_hardirqs_off_pipelined);
+
+void trace_hardirqs_on_pipelined(void)
+{
+	WARN_ON(irq_pipeline_debug() && !hard_irqs_disabled());
+
+	/*
+	 * If the in-band stage of the kernel is current but the IRQ
+	 * was not delivered because the latter is stalled, keep the
+	 * tracing logic unaware of the receipt, so that no false
+	 * positive is triggered in lockdep (e.g. IN-HARDIRQ-W ->
+	 * HARDIRQ-ON-W).
+	 */
+	if (running_inband() && !irqs_disabled()) {
+		stall_inband();
+		trace_hardirqs_on();
+		unstall_inband_nocheck();
+	}
+}
+EXPORT_SYMBOL(trace_hardirqs_on_pipelined);
+NOKPROBE_SYMBOL(trace_hardirqs_on_pipelined);
+
+#else
+
+void trace_hardirqs_off_pipelined(void)
+{
+	trace_hardirqs_off();
+}
+EXPORT_SYMBOL(trace_hardirqs_off_pipelined);
+NOKPROBE_SYMBOL(trace_hardirqs_off_pipelined);
+
+void trace_hardirqs_on_pipelined(void)
+{
+	trace_hardirqs_on();
+}
+EXPORT_SYMBOL(trace_hardirqs_on_pipelined);
+NOKPROBE_SYMBOL(trace_hardirqs_on_pipelined);
+
+#endif
+
 #endif /* CONFIG_TRACE_IRQFLAGS */
 
 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
diff --git a/kernel/kernel/trace/trace_sched_wakeup.c b/kernel/kernel/trace/trace_sched_wakeup.c
index 97b10bb31a..f7637f9c7f 100644
--- a/kernel/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/kernel/trace/trace_sched_wakeup.c
@@ -486,7 +486,9 @@ probe_wakeup_sched_switch(void *ignore, bool preempt,
 
 	if (likely(!is_tracing_stopped())) {
 		wakeup_trace->max_latency = delta;
+		hard_local_irq_disable();
 		update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu, NULL);
+		hard_local_irq_enable();
 	}
 
 out_unlock:
diff --git a/kernel/kernel/trace/trace_stack.c b/kernel/kernel/trace/trace_stack.c
index c408423e5d..16392f414d 100644
--- a/kernel/kernel/trace/trace_stack.c
+++ b/kernel/kernel/trace/trace_stack.c
@@ -171,8 +171,9 @@ static void check_stack(unsigned long ip, unsigned long *stack)
 	if (!object_is_on_stack(stack))
 		return;
 
-	/* Can't do this from NMI context (can cause deadlocks) */
-	if (in_nmi())
+	/* Can't do this from NMI or oob stage contexts (can cause
+	   deadlocks) */
+	if (in_nmi() || !running_inband())
 		return;
 
 	local_irq_save(flags);
diff --git a/kernel/lib/Kconfig.debug b/kernel/lib/Kconfig.debug
index 8608332e0e..77c03b334a 100644
--- a/kernel/lib/Kconfig.debug
+++ b/kernel/lib/Kconfig.debug
@@ -897,6 +897,38 @@ config DEBUG_SHIRQ
 	  is currently disabled). Drivers need to handle this correctly. Some
 	  don't and need to be caught.
 
+config DEBUG_IRQ_PIPELINE
+	bool "Debug IRQ pipeline"
+	depends on IRQ_PIPELINE && DEBUG_KERNEL
+	help
+	  Turn on this option for enabling debug checks related to
+	  interrupt pipelining, like interrupt state consistency and
+	  proper context isolation between the in-band and oob stages.
+
+	  If unsure, say N.
+
+config IRQ_PIPELINE_TORTURE_TEST
+	bool "Torture tests for IRQ pipeline"
+	depends on DEBUG_IRQ_PIPELINE
+	select TORTURE_TEST
+	default n
+	help
+	  This option provides a kernel module that runs torture tests
+	  on the IRQ pipeline mechanism.
+
+	  Say Y here if you want the IRQ pipeline torture tests to run
+	  when the kernel starts. Say N if you are unsure.
+
+config DEBUG_DOVETAIL
+	bool "Debug Dovetail interface"
+	depends on DOVETAIL && DEBUG_KERNEL
+	select DEBUG_IRQ_PIPELINE
+	help
+	  Turn on this option for enabling debug checks related to
+	  running a dual kernel configuration, aka dovetailing. This
+	  option implicitly enables the interrupt pipeline debugging
+	  features.
+
 menu "Debug Oops, Lockups and Hangs"
 
 config PANIC_ON_OOPS
@@ -1315,6 +1347,27 @@ config DEBUG_LOCK_ALLOC
 	 spin_lock_init()/mutex_init()/etc., or whether there is any lock
 	 held during task exit.
 
+config DEBUG_HARD_LOCKS
+	bool "Debug hard spinlocks"
+	depends on DEBUG_IRQ_PIPELINE && LOCKDEP && EXPERT
+	help
+	  Turn on this option for enabling LOCKDEP for hard spinlock
+	  types used in interrupt pipelining.
+
+	  Keep in mind that enabling such feature will ruin the
+	  latency figures for any out-of-band code, this is merely
+	  useful for proving the correctness of the locking scheme of
+	  such code without any consideration for real-time
+	  guarantees. You have been warned.
+
+	  If unsure, say N.
+
+if DEBUG_HARD_LOCKS
+comment "WARNING! DEBUG_HARD_LOCKS induces **massive** latency"
+comment "overhead for the code running on the out-of-band"
+comment "interrupt stage."
+endif
+
 config LOCKDEP
 	bool
 	depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT
diff --git a/kernel/lib/atomic64.c b/kernel/lib/atomic64.c
index e98c85a997..bf7d0409e9 100644
--- a/kernel/lib/atomic64.c
+++ b/kernel/lib/atomic64.c
@@ -25,15 +25,15 @@
  * Ensure each lock is in a separate cacheline.
  */
 static union {
-	raw_spinlock_t lock;
+	hard_spinlock_t lock;
 	char pad[L1_CACHE_BYTES];
 } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
 	[0 ... (NR_LOCKS - 1)] = {
-		.lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
+		.lock =  __HARD_SPIN_LOCK_INITIALIZER(atomic64_lock.lock),
 	},
 };
 
-static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
+static inline hard_spinlock_t *lock_addr(const atomic64_t *v)
 {
 	unsigned long addr = (unsigned long) v;
 
@@ -45,7 +45,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
 s64 atomic64_read(const atomic64_t *v)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	hard_spinlock_t *lock = lock_addr(v);
 	s64 val;
 
 	raw_spin_lock_irqsave(lock, flags);
@@ -58,7 +58,7 @@ EXPORT_SYMBOL(atomic64_read);
 void atomic64_set(atomic64_t *v, s64 i)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	hard_spinlock_t *lock = lock_addr(v);
 
 	raw_spin_lock_irqsave(lock, flags);
 	v->counter = i;
@@ -70,7 +70,7 @@ EXPORT_SYMBOL(atomic64_set);
 void atomic64_##op(s64 a, atomic64_t *v)				\
 {									\
 	unsigned long flags;						\
-	raw_spinlock_t *lock = lock_addr(v);				\
+	hard_spinlock_t *lock = lock_addr(v);				\
 									\
 	raw_spin_lock_irqsave(lock, flags);				\
 	v->counter c_op a;						\
@@ -82,7 +82,7 @@ EXPORT_SYMBOL(atomic64_##op);
 s64 atomic64_##op##_return(s64 a, atomic64_t *v)			\
 {									\
 	unsigned long flags;						\
-	raw_spinlock_t *lock = lock_addr(v);				\
+	hard_spinlock_t *lock = lock_addr(v);				\
 	s64 val;							\
 									\
 	raw_spin_lock_irqsave(lock, flags);				\
@@ -96,7 +96,7 @@ EXPORT_SYMBOL(atomic64_##op##_return);
 s64 atomic64_fetch_##op(s64 a, atomic64_t *v)				\
 {									\
 	unsigned long flags;						\
-	raw_spinlock_t *lock = lock_addr(v);				\
+	hard_spinlock_t *lock = lock_addr(v);				\
 	s64 val;							\
 									\
 	raw_spin_lock_irqsave(lock, flags);				\
@@ -133,7 +133,7 @@ ATOMIC64_OPS(xor, ^=)
 s64 atomic64_dec_if_positive(atomic64_t *v)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	hard_spinlock_t *lock = lock_addr(v);
 	s64 val;
 
 	raw_spin_lock_irqsave(lock, flags);
@@ -148,7 +148,7 @@ EXPORT_SYMBOL(atomic64_dec_if_positive);
 s64 atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	hard_spinlock_t *lock = lock_addr(v);
 	s64 val;
 
 	raw_spin_lock_irqsave(lock, flags);
@@ -163,7 +163,7 @@ EXPORT_SYMBOL(atomic64_cmpxchg);
 s64 atomic64_xchg(atomic64_t *v, s64 new)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	hard_spinlock_t *lock = lock_addr(v);
 	s64 val;
 
 	raw_spin_lock_irqsave(lock, flags);
@@ -177,7 +177,7 @@ EXPORT_SYMBOL(atomic64_xchg);
 s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 {
 	unsigned long flags;
-	raw_spinlock_t *lock = lock_addr(v);
+	hard_spinlock_t *lock = lock_addr(v);
 	s64 val;
 
 	raw_spin_lock_irqsave(lock, flags);
diff --git a/kernel/lib/dump_stack.c b/kernel/lib/dump_stack.c
index b9acd9c954..aed74e324a 100644
--- a/kernel/lib/dump_stack.c
+++ b/kernel/lib/dump_stack.c
@@ -9,9 +9,11 @@
 #include <linux/sched.h>
 #include <linux/sched/debug.h>
 #include <linux/smp.h>
+#include <linux/irqstage.h>
 #include <linux/atomic.h>
 #include <linux/kexec.h>
 #include <linux/utsname.h>
+#include <linux/hardirq.h>
 
 static char dump_stack_arch_desc_str[128];
 
@@ -56,6 +58,11 @@ void dump_stack_print_info(const char *log_lvl)
 		printk("%sHardware name: %s\n",
 		       log_lvl, dump_stack_arch_desc_str);
 
+#ifdef CONFIG_IRQ_PIPELINE
+	printk("%sIRQ stage: %s\n",
+	       log_lvl, current_irq_stage->name);
+#endif
+
 	print_worker_info(log_lvl, current);
 }
 
@@ -85,6 +92,29 @@ static void __dump_stack(const char *log_lvl)
 #ifdef CONFIG_SMP
 static atomic_t dump_lock = ATOMIC_INIT(-1);
 
+static unsigned long disable_local_irqs(void)
+{
+	unsigned long flags = 0; /* only to trick the UMR detection */
+
+	/*
+	 * We neither need nor want to disable in-band IRQs over the
+	 * oob stage, where CPU migration can't happen. Conversely, we
+	 * neither need nor want to disable hard IRQs from the oob
+	 * stage, so that latency won't skyrocket as a result of
+	 * dumping the stack backtrace.
+	 */
+	if (running_inband() && !on_pipeline_entry())
+		local_irq_save(flags);
+
+	return flags;
+}
+
+static void restore_local_irqs(unsigned long flags)
+{
+	if (running_inband() && !on_pipeline_entry())
+		local_irq_restore(flags);
+}
+
 asmlinkage __visible void dump_stack_lvl(const char *log_lvl)
 {
 	unsigned long flags;
@@ -97,7 +127,7 @@ asmlinkage __visible void dump_stack_lvl(const char *log_lvl)
 	 * against other CPUs
 	 */
 retry:
-	local_irq_save(flags);
+	flags = disable_local_irqs();
 	cpu = smp_processor_id();
 	old = atomic_cmpxchg(&dump_lock, -1, cpu);
 	if (old == -1) {
@@ -105,7 +135,7 @@ asmlinkage __visible void dump_stack_lvl(const char *log_lvl)
 	} else if (old == cpu) {
 		was_locked = 1;
 	} else {
-		local_irq_restore(flags);
+		restore_local_irqs(flags);
 		/*
 		 * Wait for the lock to release before jumping to
 		 * atomic_cmpxchg() in order to mitigate the thundering herd
@@ -120,7 +150,7 @@ asmlinkage __visible void dump_stack_lvl(const char *log_lvl)
 	if (!was_locked)
 		atomic_set(&dump_lock, -1);
 
-	local_irq_restore(flags);
+	restore_local_irqs(flags);
 }
 #else
 asmlinkage __visible void dump_stack_lvl(const char *log_lvl)
diff --git a/kernel/lib/smp_processor_id.c b/kernel/lib/smp_processor_id.c
index 2916606a93..952e2ad9ac 100644
--- a/kernel/lib/smp_processor_id.c
+++ b/kernel/lib/smp_processor_id.c
@@ -7,12 +7,16 @@
 #include <linux/export.h>
 #include <linux/kprobes.h>
 #include <linux/sched.h>
+#include <linux/irqstage.h>
 
 noinstr static
 unsigned int check_preemption_disabled(const char *what1, const char *what2)
 {
 	int this_cpu = raw_smp_processor_id();
 
+	if (hard_irqs_disabled() || !running_inband())
+		goto out;
+
 	if (likely(preempt_count()))
 		goto out;
 
diff --git a/kernel/lib/vdso/Kconfig b/kernel/lib/vdso/Kconfig
index d883ac2995..7b327e182c 100644
--- a/kernel/lib/vdso/Kconfig
+++ b/kernel/lib/vdso/Kconfig
@@ -30,4 +30,12 @@ config GENERIC_VDSO_TIME_NS
 	  Selected by architectures which support time namespaces in the
 	  VDSO
 
+config GENERIC_CLOCKSOURCE_VDSO
+        depends on ARM || ARM64
+        select CLKSRC_MMIO
+	bool
+	help
+	   Enables access to clocksources via the vDSO based on
+	   generic MMIO operations.
+
 endif
diff --git a/kernel/lib/vdso/gettimeofday.c b/kernel/lib/vdso/gettimeofday.c
index c6f6dee087..57a562705a 100644
--- a/kernel/lib/vdso/gettimeofday.c
+++ b/kernel/lib/vdso/gettimeofday.c
@@ -5,6 +5,245 @@
 #include <vdso/datapage.h>
 #include <vdso/helpers.h>
 
+static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
+			struct __kernel_timespec *ts);
+
+#ifndef vdso_clocksource_ok
+static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
+{
+	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
+}
+#endif
+
+#ifndef vdso_cycles_ok
+static inline bool vdso_cycles_ok(u64 cycles)
+{
+	return true;
+}
+#endif
+
+#if defined(CONFIG_GENERIC_CLOCKSOURCE_VDSO) && !defined(BUILD_VDSO32)
+
+#include <linux/fcntl.h>
+#include <linux/io.h>
+#include <linux/ioctl.h>
+#include <uapi/linux/clocksource.h>
+
+static notrace u64 readl_mmio_up(const struct clksrc_info *vinfo)
+{
+	const struct clksrc_user_mmio_info *info = &vinfo->mmio;
+	return readl_relaxed(info->reg_lower);
+}
+
+static notrace u64 readl_mmio_down(const struct clksrc_info *vinfo)
+{
+	const struct clksrc_user_mmio_info *info = &vinfo->mmio;
+	return ~(u64)readl_relaxed(info->reg_lower) & info->mask_lower;
+}
+
+static notrace u64 readw_mmio_up(const struct clksrc_info *vinfo)
+{
+	const struct clksrc_user_mmio_info *info = &vinfo->mmio;
+	return readw_relaxed(info->reg_lower);
+}
+
+static notrace u64 readw_mmio_down(const struct clksrc_info *vinfo)
+{
+	const struct clksrc_user_mmio_info *info = &vinfo->mmio;
+	return ~(u64)readl_relaxed(info->reg_lower) & info->mask_lower;
+}
+
+static notrace u64 readl_dmmio_up(const struct clksrc_info *vinfo)
+{
+	const struct clksrc_user_mmio_info *info = &vinfo->mmio;
+	void __iomem *reg_lower, *reg_upper;
+	u32 upper, old_upper, lower;
+
+	reg_lower = info->reg_lower;
+	reg_upper = info->reg_upper;
+
+	upper = readl_relaxed(reg_upper);
+	do {
+		old_upper = upper;
+		lower = readl_relaxed(reg_lower);
+		upper = readl_relaxed(reg_upper);
+	} while (upper != old_upper);
+
+	return (((u64)upper) << info->bits_lower) | lower;
+}
+
+static notrace u64 readw_dmmio_up(const struct clksrc_info *vinfo)
+{
+	const struct clksrc_user_mmio_info *info = &vinfo->mmio;
+	void __iomem *reg_lower, *reg_upper;
+	u16 upper, old_upper, lower;
+
+	reg_lower = info->reg_lower;
+	reg_upper = info->reg_upper;
+
+	upper = readw_relaxed(reg_upper);
+	do {
+		old_upper = upper;
+		lower = readw_relaxed(reg_lower);
+		upper = readw_relaxed(reg_upper);
+	} while (upper != old_upper);
+
+	return (((u64)upper) << info->bits_lower) | lower;
+}
+
+static notrace __cold vdso_read_cycles_t *get_mmio_read_cycles(unsigned int type)
+{
+	switch (type) {
+	case CLKSRC_MMIO_L_UP:
+		return &readl_mmio_up;
+	case CLKSRC_MMIO_L_DOWN:
+		return &readl_mmio_down;
+	case CLKSRC_MMIO_W_UP:
+		return &readw_mmio_up;
+	case CLKSRC_MMIO_W_DOWN:
+		return &readw_mmio_down;
+	case CLKSRC_DMMIO_L_UP:
+		return &readl_dmmio_up;
+	case CLKSRC_DMMIO_W_UP:
+		return &readw_dmmio_up;
+	default:
+		return NULL;
+	}
+}
+
+static __always_inline u16 to_cs_type(u32 cs_type_seq)
+{
+	return cs_type_seq >> 16;
+}
+
+static __always_inline u16 to_seq(u32 cs_type_seq)
+{
+	return cs_type_seq;
+}
+
+static __always_inline u32 to_cs_type_seq(u16 type, u16 seq)
+{
+	return (u32)type << 16U | seq;
+}
+
+static notrace noinline __cold
+void map_clocksource(const struct vdso_data *vd, struct vdso_priv *vp,
+		     u32 seq, u32 new_cs_type_seq)
+{
+	vdso_read_cycles_t *read_cycles = NULL;
+	u32 new_cs_seq, new_cs_type;
+	struct clksrc_info *info;
+	int fd, ret;
+
+	new_cs_seq = to_seq(new_cs_type_seq);
+	new_cs_type = to_cs_type(new_cs_type_seq);
+	info = &vp->clksrc_info[new_cs_type];
+
+	if (new_cs_type < CLOCKSOURCE_VDSO_MMIO)
+		goto done;
+
+	fd = clock_open_device(vd->cs_mmdev, O_RDONLY);
+	if (fd < 0)
+		goto fallback_to_syscall;
+
+	if (vdso_read_retry(vd, seq)) {
+		vdso_read_begin(vd);
+		if (to_seq(vd->cs_type_seq) != new_cs_seq) {
+			/*
+			 * cs_mmdev no longer corresponds to
+			 * vd->cs_type_seq.
+			 */
+			clock_close_device(fd);
+			return;
+		}
+	}
+
+	ret = clock_ioctl_device(fd, CLKSRC_USER_MMIO_MAP, (long)&info->mmio);
+	clock_close_device(fd);
+	if (ret < 0)
+		goto fallback_to_syscall;
+
+	read_cycles = get_mmio_read_cycles(info->mmio.type);
+	if (read_cycles == NULL) /* Mmhf, misconfigured. */
+		goto fallback_to_syscall;
+done:
+	info->read_cycles = read_cycles;
+	smp_wmb();
+	new_cs_type_seq = to_cs_type_seq(new_cs_type, new_cs_seq);
+	WRITE_ONCE(vp->current_cs_type_seq, new_cs_type_seq);
+
+	return;
+
+fallback_to_syscall:
+	new_cs_type = CLOCKSOURCE_VDSO_NONE;
+	info = &vp->clksrc_info[new_cs_type];
+	goto done;
+}
+
+static inline notrace
+bool get_hw_counter(const struct vdso_data *vd, u32 *r_seq, u64 *cycles)
+{
+	const struct clksrc_info *info;
+	struct vdso_priv *vp;
+	u32 seq, cs_type_seq;
+	unsigned int cs;
+
+	vp = __arch_get_vdso_priv();
+
+	for (;;) {
+		seq = vdso_read_begin(vd);
+		cs_type_seq = READ_ONCE(vp->current_cs_type_seq);
+		if (likely(to_seq(cs_type_seq) == to_seq(vd->cs_type_seq)))
+			break;
+
+		map_clocksource(vd, vp, seq, vd->cs_type_seq);
+	}
+
+	switch (to_cs_type(cs_type_seq)) {
+	case CLOCKSOURCE_VDSO_NONE:
+		return false; /* Use fallback. */
+	case CLOCKSOURCE_VDSO_ARCHITECTED:
+		if (unlikely(!vdso_clocksource_ok(vd)))
+			return false;
+		*cycles = __arch_get_hw_counter(vd->clock_mode, vd);
+		if (unlikely(!vdso_cycles_ok(*cycles)))
+			return false;
+		break;
+	default:
+		cs = to_cs_type(READ_ONCE(cs_type_seq));
+		info = &vp->clksrc_info[cs];
+		*cycles = info->read_cycles(info);
+		break;
+	}
+
+	*r_seq = seq;
+
+	return true;
+}
+
+#else
+
+static inline notrace
+bool get_hw_counter(const struct vdso_data *vd, u32 *r_seq, u64 *cycles)
+{
+	*r_seq = vdso_read_begin(vd);
+
+	/*
+	 * CAUTION: checking the clocksource mode must happen inside
+	 * the seqlocked section.
+	 */
+	if (unlikely(!vdso_clocksource_ok(vd)))
+		return false;
+
+	*cycles = __arch_get_hw_counter(vd->clock_mode, vd);
+	if (unlikely(!vdso_cycles_ok(*cycles)))
+		  return false;
+
+	return true;
+}
+
+#endif /* CONFIG_GENERIC_CLOCKSOURCE_VDSO */
+
 #ifndef vdso_calc_delta
 /*
  * Default implementation which works for all sane clocksources. That
@@ -31,20 +270,6 @@ static inline bool __arch_vdso_hres_capable(void)
 }
 #endif
 
-#ifndef vdso_clocksource_ok
-static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
-{
-	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
-}
-#endif
-
-#ifndef vdso_cycles_ok
-static inline bool vdso_cycles_ok(u64 cycles)
-{
-	return true;
-}
-#endif
-
 #ifdef CONFIG_TIME_NS
 static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
 					  struct __kernel_timespec *ts)
@@ -63,13 +288,7 @@ static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_
 	vdso_ts = &vd->basetime[clk];
 
 	do {
-		seq = vdso_read_begin(vd);
-
-		if (unlikely(!vdso_clocksource_ok(vd)))
-			return -1;
-
-		cycles = __arch_get_hw_counter(vd->clock_mode, vd);
-		if (unlikely(!vdso_cycles_ok(cycles)))
+		if (!get_hw_counter(vd, &seq, &cycles))
 			return -1;
 		ns = vdso_ts->nsec;
 		last = vd->cycle_last;
@@ -117,30 +336,29 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
 
 	do {
 		/*
-		 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
-		 * enabled tasks have a special VVAR page installed which
-		 * has vd->seq set to 1 and vd->clock_mode set to
-		 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
-		 * this does not affect performance because if vd->seq is
-		 * odd, i.e. a concurrent update is in progress the extra
+		 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time
+		 * namespace enabled tasks have a special VVAR page
+		 * installed which has vd->seq set to 1 and
+		 * vd->clock_mode set to VDSO_CLOCKMODE_TIMENS. For
+		 * non time namespace affected tasks this does not
+		 * affect performance because if vd->seq is odd,
+		 * i.e. a concurrent update is in progress the extra
 		 * check for vd->clock_mode is just a few extra
-		 * instructions while spin waiting for vd->seq to become
-		 * even again.
+		 * instructions while spin waiting for vd->seq to
+		 * become even again.
 		 */
 		while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
 			if (IS_ENABLED(CONFIG_TIME_NS) &&
-			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
+				vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
 				return do_hres_timens(vd, clk, ts);
 			cpu_relax();
 		}
+
 		smp_rmb();
 
-		if (unlikely(!vdso_clocksource_ok(vd)))
+		if (!get_hw_counter(vd, &seq, &cycles))
 			return -1;
 
-		cycles = __arch_get_hw_counter(vd->clock_mode, vd);
-		if (unlikely(!vdso_cycles_ok(cycles)))
-			return -1;
 		ns = vdso_ts->nsec;
 		last = vd->cycle_last;
 		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
diff --git a/kernel/mm/ioremap.c b/kernel/mm/ioremap.c
index 5fa1ab41d1..4071aa6817 100644
--- a/kernel/mm/ioremap.c
+++ b/kernel/mm/ioremap.c
@@ -241,6 +241,7 @@ int ioremap_page_range(unsigned long addr,
 			break;
 	} while (pgd++, phys_addr += (next - addr), addr = next, addr != end);
 
+	arch_advertise_page_mapping(start, end);
 	flush_cache_vmap(start, end);
 
 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
diff --git a/kernel/mm/kasan/report.c b/kernel/mm/kasan/report.c
index 8fff1825b2..ea779a40ac 100644
--- a/kernel/mm/kasan/report.c
+++ b/kernel/mm/kasan/report.c
@@ -73,7 +73,7 @@ static void print_error_description(struct kasan_access_info *info)
 			info->access_addr, current->comm, task_pid_nr(current));
 }
 
-static DEFINE_SPINLOCK(report_lock);
+static DEFINE_HARD_SPINLOCK(report_lock);
 
 static void start_report(unsigned long *flags)
 {
@@ -81,7 +81,7 @@ static void start_report(unsigned long *flags)
 	 * Make sure we don't end up in loop.
 	 */
 	kasan_disable_current();
-	spin_lock_irqsave(&report_lock, *flags);
+	raw_spin_lock_irqsave(&report_lock, *flags);
 	pr_err("==================================================================\n");
 }
 
@@ -91,7 +91,7 @@ static void end_report(unsigned long *flags, unsigned long addr)
 		trace_error_report_end(ERROR_DETECTOR_KASAN, addr);
 	pr_err("==================================================================\n");
 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
-	spin_unlock_irqrestore(&report_lock, *flags);
+	raw_spin_unlock_irqrestore(&report_lock, *flags);
 	if (panic_on_warn && !test_bit(KASAN_BIT_MULTI_SHOT, &kasan_flags)) {
 		/*
 		 * This thread may hit another WARN() in the panic path.
diff --git a/kernel/mm/memory.c b/kernel/mm/memory.c
index 834078ab6e..51ec122d7c 100644
--- a/kernel/mm/memory.c
+++ b/kernel/mm/memory.c
@@ -844,6 +844,14 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
 	if (!is_cow_mapping(src_vma->vm_flags))
 		return 1;
 
+	/*
+	 * If the source mm belongs to a Dovetail-enabled process, we
+	 * don't want to impose the COW-induced latency on it: make
+	 * sure the child gets its own copy of the page.
+	 */
+	if (dovetailing() && test_bit(MMF_DOVETAILED, &src_mm->flags))
+		goto do_copy;
+
 	/*
 	 * What we want to do is to check whether this page may
 	 * have been pinned by the parent process.  If so,
@@ -862,6 +870,7 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
 	if (likely(!page_maybe_dma_pinned(page)))
 		return 1;
 
+do_copy:
 	/*
 	 * The vma->anon_vma of the child process may be NULL
 	 * because the entire vma does not contain anonymous pages.
@@ -5695,6 +5704,15 @@ void print_vma_addr(char *prefix, unsigned long ip)
 #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
 void __might_fault(const char *file, int line)
 {
+	/*
+	 * When running over the oob stage (e.g. some co-kernel's own
+	 * thread), we should only make sure to run with hw IRQs
+	 * enabled before accessing the memory.
+	 */
+	if (running_oob()) {
+		WARN_ON_ONCE(hard_irqs_disabled());
+		return;
+	}
 	/*
 	 * Some code (nfs/sunrpc) uses socket ops on kernel memory while
 	 * holding the mmap_lock, this is safe because kernel memory doesn't
diff --git a/kernel/mm/mprotect.c b/kernel/mm/mprotect.c
index c1c3315c44..2015be98fd 100644
--- a/kernel/mm/mprotect.c
+++ b/kernel/mm/mprotect.c
@@ -41,7 +41,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 {
 	pte_t *pte, oldpte;
 	spinlock_t *ptl;
-	unsigned long pages = 0;
+	unsigned long pages = 0, flags;
 	int target_node = NUMA_NO_NODE;
 	bool dirty_accountable = cp_flags & MM_CP_DIRTY_ACCT;
 	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
@@ -113,6 +113,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 					continue;
 			}
 
+			flags = hard_local_irq_save();
 			oldpte = ptep_modify_prot_start(vma, addr, pte);
 			ptent = pte_modify(oldpte, newprot);
 			if (preserve_write)
@@ -138,6 +139,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 				ptent = pte_mkwrite(ptent);
 			}
 			ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
+			hard_local_irq_restore(flags);
 			pages++;
 		} else if (is_swap_pte(oldpte)) {
 			swp_entry_t entry = pte_to_swp_entry(oldpte);
diff --git a/kernel/mm/vmalloc.c b/kernel/mm/vmalloc.c
index 3b56c30a8e..7fccdee822 100644
--- a/kernel/mm/vmalloc.c
+++ b/kernel/mm/vmalloc.c
@@ -272,6 +272,10 @@ static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
 	return 0;
 }
 
+void __weak arch_advertise_page_mapping(unsigned long start, unsigned long end)
+{
+}
+
 /**
  * map_kernel_range_noflush - map kernel VM area with the specified pages
  * @addr: start of the VM area to map
@@ -315,6 +319,8 @@ int map_kernel_range_noflush(unsigned long addr, unsigned long size,
 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
 		arch_sync_kernel_mappings(start, end);
 
+	arch_advertise_page_mapping(start, end);
+
 	return 0;
 }
 
diff --git a/kernel/net/Kconfig b/kernel/net/Kconfig
index d6567162c1..0d39d1f56c 100644
--- a/kernel/net/Kconfig
+++ b/kernel/net/Kconfig
@@ -58,6 +58,9 @@ config NET_REDIRECT
 config SKB_EXTENSIONS
 	bool
 
+config NET_OOB
+	bool
+
 menu "Networking options"
 
 source "net/packet/Kconfig"
diff --git a/kernel/net/core/dev.c b/kernel/net/core/dev.c
index bc5dcf57eb..01d2396278 100644
--- a/kernel/net/core/dev.c
+++ b/kernel/net/core/dev.c
@@ -3111,6 +3111,10 @@ void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
 	} else if (likely(!refcount_dec_and_test(&skb->users))) {
 		return;
 	}
+
+	if (recycle_oob_skb(skb))
+		return;
+
 	get_kfree_skb_cb(skb)->reason = reason;
 	local_irq_save(flags);
 	skb->next = __this_cpu_read(softnet_data.completion_queue);
@@ -3584,7 +3588,12 @@ static int xmit_one(struct sk_buff *skb, struct net_device *dev,
 	unsigned int len;
 	int rc;
 
-	if (dev_nit_active(dev))
+	/*
+	 * Clone-relay outgoing packet to listening taps. Network taps
+	 * interested in out-of-band traffic should be handled by the
+	 * companion core.
+	 */
+	if (dev_nit_active(dev) && !skb_is_oob(skb))
 		dev_queue_xmit_nit(skb, dev);
 
 	len = skb->len;
@@ -4797,6 +4806,81 @@ int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
 }
 EXPORT_SYMBOL_GPL(do_xdp_generic);
 
+#ifdef CONFIG_NET_OOB
+
+__weak bool netif_oob_deliver(struct sk_buff *skb)
+{
+	return false;
+}
+
+__weak int netif_xmit_oob(struct sk_buff *skb)
+{
+	return NET_XMIT_DROP;
+}
+
+static bool netif_receive_oob(struct sk_buff *skb)
+{
+	struct net_device *dev = skb->dev;
+
+	if (dev && netif_oob_diversion(dev))
+		return netif_oob_deliver(skb);
+
+	return false;
+}
+
+static bool netif_receive_oob_list(struct list_head *head)
+{
+	struct sk_buff *skb, *next;
+	struct net_device *dev;
+
+	if (list_empty(head))
+		return false;
+
+	dev = list_first_entry(head, struct sk_buff, list)->dev;
+	if (!dev || !netif_oob_diversion(dev))
+		return false;
+
+	/* Callee dequeues every skb it consumes. */
+	list_for_each_entry_safe(skb, next, head, list)
+		netif_oob_deliver(skb);
+
+	return list_empty(head);
+}
+
+__weak void netif_oob_run(struct net_device *dev)
+{ }
+
+static void napi_complete_oob(struct napi_struct *n)
+{
+	struct net_device *dev = n->dev;
+
+	if (netif_oob_diversion(dev))
+		netif_oob_run(dev);
+}
+
+__weak void skb_inband_xmit_backlog(void)
+{ }
+
+#else
+
+static inline bool netif_receive_oob(struct sk_buff *skb)
+{
+	return false;
+}
+
+static inline bool netif_receive_oob_list(struct list_head *head)
+{
+	return false;
+}
+
+static inline void napi_complete_oob(struct napi_struct *n)
+{ }
+
+static inline void skb_inband_xmit_backlog(void)
+{ }
+
+#endif
+
 static int netif_rx_internal(struct sk_buff *skb)
 {
 	int ret;
@@ -4896,6 +4980,8 @@ static __latent_entropy void net_tx_action(struct softirq_action *h)
 {
 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
 
+	skb_inband_xmit_backlog();
+
 	if (sd->completion_queue) {
 		struct sk_buff *clist;
 
@@ -5639,6 +5725,9 @@ int netif_receive_skb(struct sk_buff *skb)
 {
 	int ret;
 
+	if (netif_receive_oob(skb))
+		return NET_RX_SUCCESS;
+
 	trace_netif_receive_skb_entry(skb);
 
 	ret = netif_receive_skb_internal(skb);
@@ -5662,6 +5751,8 @@ void netif_receive_skb_list(struct list_head *head)
 {
 	struct sk_buff *skb;
 
+	if (netif_receive_oob_list(head))
+		return;
 	if (list_empty(head))
 		return;
 	if (trace_netif_receive_skb_list_entry_enabled()) {
@@ -6152,6 +6243,9 @@ gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
 	gro_result_t ret;
 
+	if (netif_receive_oob(skb))
+		return GRO_NORMAL;
+
 	skb_mark_napi_id(skb, napi);
 	trace_napi_gro_receive_entry(skb);
 
@@ -6489,6 +6583,8 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
 	unsigned long flags, val, new, timeout = 0;
 	bool ret = true;
 
+	napi_complete_oob(n);
+
 	/*
 	 * 1) Don't let napi dequeue from the cpu poll list
 	 *    just in case its running on a different cpu.
diff --git a/kernel/net/core/net-sysfs.c b/kernel/net/core/net-sysfs.c
index 989b3f7ee8..a467fcac7a 100644
--- a/kernel/net/core/net-sysfs.c
+++ b/kernel/net/core/net-sysfs.c
@@ -386,6 +386,54 @@ static ssize_t tx_queue_len_store(struct device *dev,
 }
 NETDEVICE_SHOW_RW(tx_queue_len, fmt_dec);
 
+#ifdef CONFIG_NET_OOB
+
+__weak int netif_oob_switch_port(struct net_device *dev, bool enabled)
+{
+	return 0;
+}
+
+__weak bool netif_oob_get_port(struct net_device *dev)
+{
+	return false;
+}
+
+__weak ssize_t netif_oob_query_pool(struct net_device *dev, char *buf)
+{
+	return -EIO;
+}
+
+static int switch_oob_port(struct net_device *dev, unsigned long enable)
+{
+	return netif_oob_switch_port(dev, (bool)enable);
+}
+
+static ssize_t oob_port_store(struct device *dev, struct device_attribute *attr,
+			const char *buf, size_t len)
+{
+	return netdev_store(dev, attr, buf, len, switch_oob_port);
+}
+
+static ssize_t oob_port_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	struct net_device *netdev = to_net_dev(dev);
+
+	return sprintf(buf, fmt_dec, netif_oob_get_port(netdev));
+}
+static DEVICE_ATTR_RW(oob_port);
+
+static ssize_t oob_pool_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	struct net_device *netdev = to_net_dev(dev);
+
+	return netif_oob_query_pool(netdev, buf);
+}
+static DEVICE_ATTR_RO(oob_pool);
+
+#endif
+
 static int change_gro_flush_timeout(struct net_device *dev, unsigned long val)
 {
 	WRITE_ONCE(dev->gro_flush_timeout, val);
@@ -619,6 +667,10 @@ static struct attribute *net_class_attrs[] __ro_after_init = {
 	&dev_attr_proto_down.attr,
 	&dev_attr_carrier_up_count.attr,
 	&dev_attr_carrier_down_count.attr,
+#ifdef CONFIG_NET_OOB
+	&dev_attr_oob_port.attr,
+	&dev_attr_oob_pool.attr,
+#endif
 	NULL,
 };
 ATTRIBUTE_GROUPS(net_class);
diff --git a/kernel/net/core/skbuff.c b/kernel/net/core/skbuff.c
index 382dbdc553..ba8222f939 100644
--- a/kernel/net/core/skbuff.c
+++ b/kernel/net/core/skbuff.c
@@ -291,6 +291,108 @@ static struct sk_buff *__build_skb_around(struct sk_buff *skb,
 	return skb;
 }
 
+#ifdef CONFIG_NET_OOB
+
+struct sk_buff *__netdev_alloc_oob_skb(struct net_device *dev, size_t len,
+				size_t headroom, gfp_t gfp_mask)
+{
+	struct sk_buff *skb;
+
+	headroom = ALIGN(NET_SKB_PAD + headroom, NET_SKB_PAD);
+	skb = __alloc_skb(len + headroom, gfp_mask,
+			SKB_ALLOC_RX, NUMA_NO_NODE);
+	if (!skb)
+		return NULL;
+
+	skb_reserve(skb, headroom);
+	skb->dev = dev;
+	skb->oob = true;
+
+	return skb;
+}
+EXPORT_SYMBOL_GPL(__netdev_alloc_oob_skb);
+
+void __netdev_free_oob_skb(struct net_device *dev, struct sk_buff *skb)
+{
+	skb->oob = false;
+	skb->oob_clone = false;
+	dev_kfree_skb(skb);
+}
+EXPORT_SYMBOL_GPL(__netdev_free_oob_skb);
+
+void netdev_reset_oob_skb(struct net_device *dev, struct sk_buff *skb,
+			size_t headroom)
+{
+	unsigned char *data = skb->head; /* Always from kmalloc_reserve(). */
+
+	if (WARN_ON_ONCE(!skb->oob || skb->oob_clone))
+		return;
+
+	memset(skb, 0, offsetof(struct sk_buff, tail));
+	__build_skb_around(skb, data, 0);
+	headroom = ALIGN(NET_SKB_PAD + headroom, NET_SKB_PAD);
+	skb_reserve(skb, headroom);
+	skb->oob = true;
+	skb->dev = dev;
+}
+EXPORT_SYMBOL_GPL(netdev_reset_oob_skb);
+
+struct sk_buff *skb_alloc_oob_head(gfp_t gfp_mask)
+{
+	struct sk_buff *skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
+
+	if (!skb)
+		return NULL;
+
+	/*
+	 * skb heads allocated for out-of-band traffic should be
+	 * reserved for clones, so memset is extraneous in the sense
+	 * that skb_morph_oob() should follow the allocation.
+	 */
+	memset(skb, 0, offsetof(struct sk_buff, tail));
+	refcount_set(&skb->users, 1);
+	skb->oob_clone = true;
+	skb_set_kcov_handle(skb, kcov_common_handle());
+
+	return skb;
+}
+EXPORT_SYMBOL_GPL(skb_alloc_oob_head);
+
+static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb);
+
+void skb_morph_oob_skb(struct sk_buff *n, struct sk_buff *skb)
+{
+	__skb_clone(n, skb);
+	n->oob = true;
+	n->oob_clone = true;
+	skb->oob_cloned = true;
+}
+EXPORT_SYMBOL_GPL(skb_morph_oob_skb);
+
+bool skb_release_oob_skb(struct sk_buff *skb, int *dref)
+{
+	struct skb_shared_info *shinfo = skb_shinfo(skb);
+
+	if (!skb_unref(skb))
+		return false;
+
+	/*
+	 * ->nohdr is never set for oob shells, so we always refcount
+         * the full data (header + payload) when cloned.
+	 */
+	*dref = skb->cloned ? atomic_sub_return(1, &shinfo->dataref) : 0;
+
+	return true;
+}
+EXPORT_SYMBOL_GPL(skb_release_oob_skb);
+
+__weak bool skb_oob_recycle(struct sk_buff *skb)
+{
+	return false;
+}
+
+#endif	/* CONFIG_NET_OOB */
+
 /**
  * __build_skb - build a network buffer
  * @data: data buffer provided by caller
@@ -691,6 +793,9 @@ static void skb_release_all(struct sk_buff *skb)
 
 void __kfree_skb(struct sk_buff *skb)
 {
+	if (recycle_oob_skb(skb))
+		return;
+
 	skb_release_all(skb);
 	kfree_skbmem(skb);
 }
@@ -884,6 +989,9 @@ static inline void _kfree_skb_defer(struct sk_buff *skb)
 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
 
 	/* drop skb->head and call any destructors for packet */
+	if (recycle_oob_skb(skb))
+		return;
+
 	skb_release_all(skb);
 
 	/* record skb to CPU local list */
@@ -903,6 +1011,9 @@ static inline void _kfree_skb_defer(struct sk_buff *skb)
 }
 void __kfree_skb_defer(struct sk_buff *skb)
 {
+	if (recycle_oob_skb(skb))
+		return;
+
 	_kfree_skb_defer(skb);
 }
 
@@ -926,6 +1037,9 @@ void napi_consume_skb(struct sk_buff *skb, int budget)
 		return;
 	}
 
+	if (recycle_oob_skb(skb))
+		return;
+
 	_kfree_skb_defer(skb);
 }
 EXPORT_SYMBOL(napi_consume_skb);
@@ -946,6 +1060,7 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
 	skb_dst_copy(new, old);
 	__skb_ext_copy(new, old);
 	__nf_copy(new, old, false);
+	__skb_oob_copy(new, old);
 
 	/* Note : this field could be in headers_start/headers_end section
 	 * It is not yet because we do not want to have a 16 bit hole
diff --git a/kernel/net/packet/af_packet.c b/kernel/net/packet/af_packet.c
index eaa030e2ad..9a19b4ae95 100644
--- a/kernel/net/packet/af_packet.c
+++ b/kernel/net/packet/af_packet.c
@@ -3309,6 +3309,7 @@ static int packet_create(struct net *net, struct socket *sock, int protocol,
 	po = pkt_sk(sk);
 	init_completion(&po->skb_completion);
 	sk->sk_family = PF_PACKET;
+	sk->sk_protocol	= protocol;
 	po->num = proto;
 	po->xmit = dev_queue_xmit;
 
diff --git a/kernel/net/sched/Kconfig b/kernel/net/sched/Kconfig
index bc4e5da76f..1bcf1b9956 100644
--- a/kernel/net/sched/Kconfig
+++ b/kernel/net/sched/Kconfig
@@ -117,6 +117,29 @@ config NET_SCH_MULTIQ
 	  To compile this code as a module, choose M here: the
 	  module will be called sch_multiq.
 
+config NET_SCH_OOB
+	tristate "Out-of-band packet queuing (OOB)"
+	depends on NET_OOB
+	help
+	  Say Y here if you want to use a Dovetail-aware packet
+	  scheduler for prioritizing egress traffic between the
+	  regular (in-band) network stack and a companion core. This
+	  scheduler helps in two cases:
+
+	  - for sending high priority packets originating from the
+	    out-of-band stage to NICs which cannot handle outgoing
+	    packets from that stage directly. In this case, these
+	    packets take precedence over regular traffic for
+	    transmission.
+
+	  - for sharing an out-of-band capable interface between the
+            in-band and out-of-band network stacks, proxying regular
+            traffic originating from the in-band stage to NICs which
+            will be processing all packets from the out-of-band stage.
+
+	  To compile this code as a module, choose M here: the
+	  module will be called sch_oob.
+
 config NET_SCH_RED
 	tristate "Random Early Detection (RED)"
 	help
diff --git a/kernel/net/sched/Makefile b/kernel/net/sched/Makefile
index 66bbf9a98f..20fc082d71 100644
--- a/kernel/net/sched/Makefile
+++ b/kernel/net/sched/Makefile
@@ -45,6 +45,7 @@ obj-$(CONFIG_NET_SCH_TBF)	+= sch_tbf.o
 obj-$(CONFIG_NET_SCH_TEQL)	+= sch_teql.o
 obj-$(CONFIG_NET_SCH_PRIO)	+= sch_prio.o
 obj-$(CONFIG_NET_SCH_MULTIQ)	+= sch_multiq.o
+obj-$(CONFIG_NET_SCH_OOB)	+= sch_oob.o
 obj-$(CONFIG_NET_SCH_ATM)	+= sch_atm.o
 obj-$(CONFIG_NET_SCH_NETEM)	+= sch_netem.o
 obj-$(CONFIG_NET_SCH_DRR)	+= sch_drr.o
diff --git a/kernel/net/sched/sch_oob.c b/kernel/net/sched/sch_oob.c
new file mode 100644
index 0000000000..22373e81bb
--- /dev/null
+++ b/kernel/net/sched/sch_oob.c
@@ -0,0 +1,294 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2020 Philippe Gerum  <rpm@xenomai.org>
+ */
+
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+#include <net/pkt_sched.h>
+#include <net/pkt_cls.h>
+
+/*
+ * With Qdisc[2], 0=oob_fallback and 1=inband. User can graft whatever
+ * qdisc on these slots; both preset to pfifo_ops. skb->oob is checked
+ * to determine which qdisc should handle the packet eventually.
+ */
+
+struct oob_qdisc_priv {
+	struct Qdisc *qdisc[2];	/* 0=oob_fallback, 1=in-band */
+	struct tcf_proto __rcu *filter_list;
+	struct tcf_block *block;
+};
+
+static int oob_enqueue(struct sk_buff *skb, struct Qdisc *sch,
+		struct sk_buff **to_free)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+	struct net_device *dev = skb->dev;
+	struct Qdisc *qdisc;
+	int ret;
+
+	/*
+	 * If the device accepts oob traffic and can handle it
+	 * directly from the oob stage, pass the outgoing packet to
+	 * the transmit handler of the oob stack. This makes sure that
+	 * all traffic, including the in-band one, flows through the
+	 * oob stack which may implement its own queuing discipline.
+	 *
+	 * netif_xmit_oob() might fail handling the packet, in which
+	 * case we leave it to the in-band packet scheduler, applying
+	 * a best-effort strategy by giving higher priority to oob
+	 * packets over mere in-band traffic.
+	 */
+	if (dev && netif_oob_diversion(dev) && netdev_is_oob_capable(dev)) {
+		ret = netif_xmit_oob(skb);
+		if (ret == NET_XMIT_SUCCESS)
+			return NET_XMIT_SUCCESS;
+	}
+
+	/*
+	 * Out-of-band fast lane is closed. Best effort: use a special
+	 * 'high priority' queue for oob packets we handle from
+	 * in-band context the usual way through the common stack.
+	 */
+	qdisc = skb->oob ? p->qdisc[0] : p->qdisc[1];
+	ret = qdisc_enqueue(skb, qdisc, to_free);
+	if (ret == NET_XMIT_SUCCESS) {
+		sch->q.qlen++;
+		return NET_XMIT_SUCCESS;
+	}
+
+	if (net_xmit_drop_count(ret))
+		qdisc_qstats_drop(sch);
+
+	return ret;
+}
+
+static struct sk_buff *oob_dequeue(struct Qdisc *sch)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+	struct sk_buff *skb;
+	struct Qdisc *qdisc;
+	int band;
+
+	/*
+	 * First try to dequeue pending out-of-band packets. If none,
+	 * then check for in-band traffic.
+	 */
+	for (band = 0; band < 2; band++) {
+		qdisc = p->qdisc[band];
+		skb = qdisc->dequeue(qdisc);
+		if (skb) {
+			qdisc_bstats_update(sch, skb);
+			sch->q.qlen--;
+			return skb;
+		}
+	}
+
+	return NULL;
+}
+
+static struct sk_buff *oob_peek(struct Qdisc *sch)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+	struct sk_buff *skb;
+	struct Qdisc *qdisc;
+	int band;
+
+	for (band = 0; band < 2; band++) {
+		qdisc = p->qdisc[band];
+		skb = qdisc->ops->peek(qdisc);
+		if (skb)
+			return skb;
+	}
+
+	return NULL;
+}
+
+static int oob_init(struct Qdisc *sch, struct nlattr *opt,
+		struct netlink_ext_ack *extack)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+	int ret;
+
+	ret = tcf_block_get(&p->block, &p->filter_list, sch, extack);
+	if (ret)
+		return ret;
+
+	p->qdisc[0] = qdisc_create_dflt(sch->dev_queue,
+					&pfifo_qdisc_ops, sch->handle,
+					extack);
+	p->qdisc[1] = qdisc_create_dflt(sch->dev_queue,
+					&pfifo_fast_ops, sch->handle,
+					extack);
+
+	return 0;
+}
+
+static void oob_reset(struct Qdisc *sch)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+
+	qdisc_reset(p->qdisc[0]);
+	qdisc_reset(p->qdisc[1]);
+	sch->q.qlen = 0;
+}
+
+static void oob_destroy(struct Qdisc *sch)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+
+	tcf_block_put(p->block);
+	qdisc_put(p->qdisc[0]);
+	qdisc_put(p->qdisc[1]);
+}
+
+static int oob_tune(struct Qdisc *sch, struct nlattr *opt,
+		struct netlink_ext_ack *extack)
+{
+	return 0;
+}
+
+static int oob_dump(struct Qdisc *sch, struct sk_buff *skb)
+{
+	return skb->len;
+}
+
+static int oob_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
+		struct Qdisc **old, struct netlink_ext_ack *extack)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+	unsigned long band = arg - 1;
+
+	if (new == NULL)
+		new = &noop_qdisc;
+
+	*old = qdisc_replace(sch, new, &p->qdisc[band]);
+
+	return 0;
+}
+
+static struct Qdisc *
+oob_leaf(struct Qdisc *sch, unsigned long arg)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+	unsigned long band = arg - 1;
+
+	return p->qdisc[band];
+}
+
+static unsigned long oob_find(struct Qdisc *sch, u32 classid)
+{
+	unsigned long band = TC_H_MIN(classid);
+
+	return band - 1 >= 2 ? 0 : band;
+}
+
+static int oob_dump_class(struct Qdisc *sch, unsigned long cl,
+			struct sk_buff *skb, struct tcmsg *tcm)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+
+	tcm->tcm_handle |= TC_H_MIN(cl);
+	tcm->tcm_info = p->qdisc[cl - 1]->handle;
+
+	return 0;
+}
+
+static int oob_dump_class_stats(struct Qdisc *sch, unsigned long cl,
+				struct gnet_dump *d)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+	struct Qdisc *cl_q = p->qdisc[cl - 1];
+
+	if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
+				  d, cl_q->cpu_bstats, &cl_q->bstats) < 0 ||
+	    qdisc_qstats_copy(d, cl_q) < 0)
+		return -1;
+
+	return 0;
+}
+
+static void oob_walk(struct Qdisc *sch, struct qdisc_walker *arg)
+{
+	int band;
+
+	if (arg->stop)
+		return;
+
+	for (band = 0; band < 2; band++) {
+		if (arg->count < arg->skip) {
+			arg->count++;
+			continue;
+		}
+		if (arg->fn(sch, band + 1, arg) < 0) {
+			arg->stop = 1;
+			break;
+		}
+		arg->count++;
+	}
+}
+
+static unsigned long oob_tcf_bind(struct Qdisc *sch, unsigned long parent,
+				 u32 classid)
+{
+	return oob_find(sch, classid);
+}
+
+static void oob_tcf_unbind(struct Qdisc *q, unsigned long cl)
+{
+}
+
+static struct tcf_block *oob_tcf_block(struct Qdisc *sch, unsigned long cl,
+				       struct netlink_ext_ack *extack)
+{
+	struct oob_qdisc_priv *p = qdisc_priv(sch);
+
+	if (cl)
+		return NULL;
+
+	return p->block;
+}
+
+static const struct Qdisc_class_ops oob_class_ops = {
+	.graft		=	oob_graft,
+	.leaf		=	oob_leaf,
+	.find		=	oob_find,
+	.walk		=	oob_walk,
+	.dump		=	oob_dump_class,
+	.dump_stats	=	oob_dump_class_stats,
+	.tcf_block	=	oob_tcf_block,
+	.bind_tcf	=	oob_tcf_bind,
+	.unbind_tcf	=	oob_tcf_unbind,
+};
+
+static struct Qdisc_ops oob_qdisc_ops __read_mostly = {
+	.cl_ops		=	&oob_class_ops,
+	.id		=	"oob",
+	.priv_size	=	sizeof(struct oob_qdisc_priv),
+	.enqueue	=	oob_enqueue,
+	.dequeue	=	oob_dequeue,
+	.peek		=	oob_peek,
+	.init		=	oob_init,
+	.reset		=	oob_reset,
+	.destroy	=	oob_destroy,
+	.change		=	oob_tune,
+	.dump		=	oob_dump,
+	.owner		=	THIS_MODULE,
+};
+
+static int __init oob_module_init(void)
+{
+	return register_qdisc(&oob_qdisc_ops);
+}
+
+static void __exit oob_module_exit(void)
+{
+	unregister_qdisc(&oob_qdisc_ops);
+}
+
+module_init(oob_module_init)
+module_exit(oob_module_exit)
+
+MODULE_LICENSE("GPL");
diff --git a/kernel/net/socket.c b/kernel/net/socket.c
index 938ab3a897..ac19deeee5 100644
--- a/kernel/net/socket.c
+++ b/kernel/net/socket.c
@@ -141,6 +141,95 @@ static void sock_show_fdinfo(struct seq_file *m, struct file *f)
 #define sock_show_fdinfo NULL
 #endif
 
+#ifdef CONFIG_NET_OOB
+
+static inline bool sock_oob_capable(struct socket *sock)
+{
+	return sock->sk && sock->sk->oob_data;
+}
+
+int __weak sock_oob_attach(struct socket *sock)
+{
+	return 0;
+}
+
+void __weak sock_oob_detach(struct socket *sock)
+{
+}
+
+int __weak sock_oob_bind(struct socket *sock, struct sockaddr *addr, int len)
+{
+	return 0;
+}
+
+long __weak sock_inband_ioctl_redirect(struct socket *sock,
+				unsigned int cmd, unsigned long arg)
+{
+	return -ENOTTY;
+}
+
+long __weak sock_oob_ioctl(struct file *file,
+			unsigned int cmd, unsigned long arg)
+{
+	return -ENOTTY;
+}
+
+ssize_t __weak sock_oob_write(struct file *filp,
+				const char __user *u_buf, size_t count)
+{
+	return -EOPNOTSUPP;
+}
+
+ssize_t __weak sock_oob_read(struct file *filp,
+			char __user *u_buf, size_t count)
+{
+	return -EOPNOTSUPP;
+}
+
+__poll_t __weak sock_oob_poll(struct file *filp,
+				struct oob_poll_wait *wait)
+{
+	return -EOPNOTSUPP;
+}
+
+#define compat_sock_oob_ioctl compat_ptr_oob_ioctl
+
+#else	/* !CONFIG_NET_OOB */
+
+static inline bool sock_oob_capable(struct socket *sock)
+{
+	return false;
+}
+
+static inline int sock_oob_attach(struct socket *sock)
+{
+	return 0;
+}
+
+static inline void sock_oob_detach(struct socket *sock)
+{
+}
+
+static int sock_oob_bind(struct socket *sock,
+			struct sockaddr *addr, int len)
+{
+	return 0;
+}
+
+static inline long sock_inband_ioctl_redirect(struct socket *sock,
+					unsigned int cmd, unsigned long arg)
+{
+	return -ENOTTY;
+}
+
+#define sock_oob_ioctl		NULL
+#define sock_oob_write		NULL
+#define sock_oob_read		NULL
+#define sock_oob_poll		NULL
+#define compat_sock_oob_ioctl	NULL
+
+#endif	/* !CONFIG_NET_OOB */
+
 /*
  *	Socket files have a set of 'special' operations as well as the generic file ones. These don't appear
  *	in the operation structures but are done directly via the socketcall() multiplexor.
@@ -153,8 +242,13 @@ static const struct file_operations socket_file_ops = {
 	.write_iter =	sock_write_iter,
 	.poll =		sock_poll,
 	.unlocked_ioctl = sock_ioctl,
+	.oob_ioctl =	sock_oob_ioctl,
+	.oob_write =	sock_oob_write,
+	.oob_read =	sock_oob_read,
+	.oob_poll =	sock_oob_poll,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl = compat_sock_ioctl,
+	.compat_oob_ioctl = compat_sock_oob_ioctl,
 #endif
 	.mmap =		sock_mmap,
 	.release =	sock_close,
@@ -427,7 +521,7 @@ EXPORT_SYMBOL(sock_alloc_file);
 static int sock_map_fd(struct socket *sock, int flags)
 {
 	struct file *newfile;
-	int fd = get_unused_fd_flags(flags);
+	int fd = get_unused_fd_flags(flags), ret;
 	if (unlikely(fd < 0)) {
 		sock_release(sock);
 		return fd;
@@ -435,6 +529,14 @@ static int sock_map_fd(struct socket *sock, int flags)
 
 	newfile = sock_alloc_file(sock, flags, NULL);
 	if (!IS_ERR(newfile)) {
+		if (IS_ENABLED(CONFIG_NET_OOB) && (flags & SOCK_OOB)) {
+			ret = sock_oob_attach(sock);
+			if (ret < 0) {
+				put_unused_fd(fd);
+				sock_release(sock);
+				return ret;
+			}
+		}
 		fd_install(fd, newfile);
 		return fd;
 	}
@@ -589,6 +691,9 @@ EXPORT_SYMBOL(sock_alloc);
 
 static void __sock_release(struct socket *sock, struct inode *inode)
 {
+	if (sock_oob_capable(sock))
+		sock_oob_detach(sock);
+
 	if (sock->ops) {
 		struct module *owner = sock->ops->owner;
 
@@ -1185,6 +1290,11 @@ static long sock_ioctl(struct file *file, unsigned cmd, unsigned long arg)
 						   false);
 			break;
 		default:
+			if (sock_oob_capable(sock)) {
+				err = sock_inband_ioctl_redirect(sock, cmd, arg);
+				if (!err || err != -ENOIOCTLCMD)
+					break;
+			}
 			err = sock_do_ioctl(net, sock, cmd, arg);
 			break;
 		}
@@ -1498,10 +1608,18 @@ int __sys_socket(int family, int type, int protocol)
 	BUILD_BUG_ON((SOCK_MAX | SOCK_TYPE_MASK) != SOCK_TYPE_MASK);
 	BUILD_BUG_ON(SOCK_CLOEXEC & SOCK_TYPE_MASK);
 	BUILD_BUG_ON(SOCK_NONBLOCK & SOCK_TYPE_MASK);
+	BUILD_BUG_ON(SOCK_OOB & SOCK_TYPE_MASK);
 
 	flags = type & ~SOCK_TYPE_MASK;
-	if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
+	if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK | SOCK_OOB))
 		return -EINVAL;
+	/*
+	 * Not every protocol family supports out-of-band operations,
+	 * however PF_OOB certainly does: force SOCK_OOB in, so that
+	 * sock_oob_attach() runs for this socket.
+	 */
+	if (IS_ENABLED(CONFIG_NET_OOB) && family == AF_OOB)
+		flags |= SOCK_OOB;
 	type &= SOCK_TYPE_MASK;
 
 	if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK))
@@ -1511,7 +1629,7 @@ int __sys_socket(int family, int type, int protocol)
 	if (retval < 0)
 		return retval;
 
-	return sock_map_fd(sock, flags & (O_CLOEXEC | O_NONBLOCK));
+	return sock_map_fd(sock, flags & (O_CLOEXEC | O_NONBLOCK | O_OOB));
 }
 
 SYSCALL_DEFINE3(socket, int, family, int, type, int, protocol)
@@ -1642,6 +1760,9 @@ int __sys_bind(int fd, struct sockaddr __user *umyaddr, int addrlen)
 			err = security_socket_bind(sock,
 						   (struct sockaddr *)&address,
 						   addrlen);
+			if (sock_oob_capable(sock) && !err)
+				err = sock_oob_bind(sock, (struct sockaddr *)
+						&address, addrlen);
 			if (!err)
 				err = sock->ops->bind(sock,
 						      (struct sockaddr *)
diff --git a/kernel/scripts/mkcompile_h b/kernel/scripts/mkcompile_h
index a72b154de7..5a34fe4b8e 100755
--- a/kernel/scripts/mkcompile_h
+++ b/kernel/scripts/mkcompile_h
@@ -6,8 +6,9 @@ ARCH=$2
 SMP=$3
 PREEMPT=$4
 PREEMPT_RT=$5
-CC_VERSION="$6"
-LD=$7
+IRQPIPE=$6
+CC_VERSION="$7"
+LD=$8
 
 vecho() { [ "${quiet}" = "silent_" ] || echo "$@" ; }
 
@@ -45,6 +46,7 @@ CONFIG_FLAGS=""
 if [ -n "$SMP" ] ; then CONFIG_FLAGS="SMP"; fi
 if [ -n "$PREEMPT" ] ; then CONFIG_FLAGS="$CONFIG_FLAGS PREEMPT"; fi
 if [ -n "$PREEMPT_RT" ] ; then CONFIG_FLAGS="$CONFIG_FLAGS PREEMPT_RT"; fi
+if [ -n "$IRQPIPE" ] ; then CONFIG_FLAGS="$CONFIG_FLAGS IRQPIPE"; fi
 
 # Truncate to maximum length
 UTS_LEN=64
diff --git a/kernel/security/selinux/hooks.c b/kernel/security/selinux/hooks.c
index 1235854233..765efd03ad 100644
--- a/kernel/security/selinux/hooks.c
+++ b/kernel/security/selinux/hooks.c
@@ -1280,7 +1280,9 @@ static inline u16 socket_type_to_security_class(int family, int type, int protoc
 			return SECCLASS_SMC_SOCKET;
 		case PF_XDP:
 			return SECCLASS_XDP_SOCKET;
-#if PF_MAX > 45
+		case PF_OOB:
+			return SECCLASS_OOB_SOCKET;
+#if PF_MAX > 46
 #error New address family defined, please update this function.
 #endif
 		}
diff --git a/kernel/security/selinux/include/classmap.h b/kernel/security/selinux/include/classmap.h
index 955e8c8389..79ea017563 100644
--- a/kernel/security/selinux/include/classmap.h
+++ b/kernel/security/selinux/include/classmap.h
@@ -247,6 +247,8 @@ struct security_class_mapping secclass_map[] = {
 	    NULL } },
 	{ "xdp_socket",
 	  { COMMON_SOCK_PERMS, NULL } },
+	{ "oob_socket",
+	  { COMMON_SOCK_PERMS, NULL } },
 	{ "perf_event",
 	  { "open", "cpu", "kernel", "tracepoint", "read", "write", NULL } },
 	{ "anon_inode",
@@ -254,6 +256,6 @@ struct security_class_mapping secclass_map[] = {
 	{ NULL }
   };
 
-#if PF_MAX > 45
+#if PF_MAX > 46
 #error New address family defined, please update secclass_map.
 #endif
diff --git a/kernel/tools/perf/trace/beauty/include/linux/socket.h b/kernel/tools/perf/trace/beauty/include/linux/socket.h
index 9aa530d497..93b104ee26 100644
--- a/kernel/tools/perf/trace/beauty/include/linux/socket.h
+++ b/kernel/tools/perf/trace/beauty/include/linux/socket.h
@@ -223,8 +223,9 @@ struct ucred {
 				 * reuses AF_INET address family
 				 */
 #define AF_XDP		44	/* XDP sockets			*/
+#define AF_OOB		45	/* Out-of-band domain sockets */
 
-#define AF_MAX		45	/* For now.. */
+#define AF_MAX		46	/* For now.. */
 
 /* Protocol families, same as address families. */
 #define PF_UNSPEC	AF_UNSPEC
-- 
2.17.1

