openwrt/target/linux/brcm2708/patches-3.18/0092-dwc_otg-introduce-fiq_fsm_spin-un-lock.patch

From 424f79f35a94611f73182f19a7711174b756b052 Mon Sep 17 00:00:00 2001
From: P33M <P33M@github.com>
Date: Fri, 26 Sep 2014 11:32:09 +0100
Subject: [PATCH 092/114] dwc_otg: introduce fiq_fsm_spin(un|)lock()

SMP safety for the FIQ relies on register read-modify write cycles being
completed in the correct order. Several places in the DWC code modify
registers also touched by the FIQ. Protect these by a bare-bones lock
mechanism.

This also makes it possible to run the FIQ and IRQ handlers on different
cores.
---
 .../usb/host/dwc_common_port/dwc_common_linux.c    |  6 ---
 drivers/usb/host/dwc_otg/dwc_otg_cil.c             | 10 -----
 drivers/usb/host/dwc_otg/dwc_otg_fiq_fsm.c         | 46 +++++++++++++++++++++-
 drivers/usb/host/dwc_otg/dwc_otg_fiq_fsm.h         | 16 +++++++-
 drivers/usb/host/dwc_otg/dwc_otg_hcd.c             | 23 ++++++++++-
 drivers/usb/host/dwc_otg/dwc_otg_hcd_intr.c        |  9 ++++-
 6 files changed, 88 insertions(+), 22 deletions(-)

--- a/drivers/usb/host/dwc_common_port/dwc_common_linux.c
+++ b/drivers/usb/host/dwc_common_port/dwc_common_linux.c
@@ -580,13 +580,7 @@ void DWC_WRITE_REG64(uint64_t volatile *
 
 void DWC_MODIFY_REG32(uint32_t volatile *reg, uint32_t clear_mask, uint32_t set_mask)
 {
-	unsigned long flags;
-
-	local_irq_save(flags);
-	local_fiq_disable();
 	writel((readl(reg) & ~clear_mask) | set_mask, reg);
-	local_fiq_enable();
-	local_irq_restore(flags);
 }
 
 #if 0
--- a/drivers/usb/host/dwc_otg/dwc_otg_cil.c
+++ b/drivers/usb/host/dwc_otg/dwc_otg_cil.c
@@ -2244,9 +2244,7 @@ void dwc_otg_core_host_init(dwc_otg_core
  */
 void dwc_otg_hc_init(dwc_otg_core_if_t * core_if, dwc_hc_t * hc)
 {
-	uint32_t intr_enable;
 	hcintmsk_data_t hc_intr_mask;
-	gintmsk_data_t gintmsk = {.d32 = 0 };
 	hcchar_data_t hcchar;
 	hcsplt_data_t hcsplt;
 
@@ -2348,14 +2346,6 @@ void dwc_otg_hc_init(dwc_otg_core_if_t *
 	}
 	DWC_WRITE_REG32(&hc_regs->hcintmsk, hc_intr_mask.d32);
 
-	/* Enable the top level host channel interrupt. */
-	intr_enable = (1 << hc_num);
-	DWC_MODIFY_REG32(&host_if->host_global_regs->haintmsk, 0, intr_enable);
-
-	/* Make sure host channel interrupts are enabled. */
-	gintmsk.b.hcintr = 1;
-	DWC_MODIFY_REG32(&core_if->core_global_regs->gintmsk, 0, gintmsk.d32);
-
 	/*
 	 * Program the HCCHARn register with the endpoint characteristics for
 	 * the current transfer.
--- a/drivers/usb/host/dwc_otg/dwc_otg_fiq_fsm.c
+++ b/drivers/usb/host/dwc_otg/dwc_otg_fiq_fsm.c
@@ -75,6 +75,46 @@ void notrace _fiq_print(enum fiq_debug_l
 }
 
 /**
+ * fiq_fsm_spin_lock() - ARMv6+ bare bones spinlock
+ * Must be called with local interrupts and FIQ disabled.
+ */
+inline void fiq_fsm_spin_lock(fiq_lock_t *lock)
+{
+	unsigned long tmp;
+	uint32_t newval;
+	fiq_lock_t lockval;
+	smp_mb__before_spinlock();
+	/* Nested locking, yay. If we are on the same CPU as the fiq, then the disable
+	 * will be sufficient. If we are on a different CPU, then the lock protects us. */
+	prefetchw(&lock->slock);
+	asm volatile (
+	"1:     ldrex   %0, [%3]\n"
+	"       add     %1, %0, %4\n"
+	"       strex   %2, %1, [%3]\n"
+	"       teq     %2, #0\n"
+	"       bne     1b"
+	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp)
+	: "r" (&lock->slock), "I" (1 << 16)
+	: "cc");
+
+	while (lockval.tickets.next != lockval.tickets.owner) {
+		wfe();
+		lockval.tickets.owner = ACCESS_ONCE(lock->tickets.owner);
+	}
+	smp_mb();
+}
+
+/**
+ * fiq_fsm_spin_unlock() - ARMv6+ bare bones spinunlock
+ */
+inline void fiq_fsm_spin_unlock(fiq_lock_t *lock)
+{
+	smp_mb();
+	lock->tickets.owner++;
+	dsb_sev();
+}
+
+/**
  * fiq_fsm_restart_channel() - Poke channel enable bit for a split transaction
  * @channel: channel to re-enable
  */
@@ -1142,6 +1182,7 @@ void notrace dwc_otg_fiq_fsm(struct fiq_
 	gintsts_handled.d32 = 0;
 	haint_handled.d32 = 0;
 
+	fiq_fsm_spin_lock(&state->lock);
 	gintsts.d32 = FIQ_READ(state->dwc_regs_base + GINTSTS);
 	gintmsk.d32 = FIQ_READ(state->dwc_regs_base + GINTMSK);
 	gintsts.d32 &= gintmsk.d32;
@@ -1231,7 +1272,7 @@ void notrace dwc_otg_fiq_fsm(struct fiq_
 
 	}
 	state->fiq_done++;
-	mb();
+	fiq_fsm_spin_unlock(&state->lock);
 }
 
 
@@ -1253,6 +1294,7 @@ void notrace dwc_otg_fiq_nop(struct fiq_
 	gintmsk_data_t gintmsk;
 	hfnum_data_t hfnum;
 
+	fiq_fsm_spin_lock(&state->lock);
 	hfnum.d32 = FIQ_READ(state->dwc_regs_base + HFNUM);
 	gintsts.d32 = FIQ_READ(state->dwc_regs_base + GINTSTS);
 	gintmsk.d32 = FIQ_READ(state->dwc_regs_base + GINTMSK);
@@ -1290,5 +1332,5 @@ void notrace dwc_otg_fiq_nop(struct fiq_
 
 	}
 	state->fiq_done++;
-	mb();
+	fiq_fsm_spin_unlock(&state->lock);
 }
--- a/drivers/usb/host/dwc_otg/dwc_otg_fiq_fsm.h
+++ b/drivers/usb/host/dwc_otg/dwc_otg_fiq_fsm.h
@@ -120,7 +120,6 @@ typedef struct {
 	volatile void* intstat;
 } mphi_regs_t;
 
-
 enum fiq_debug_level {
 	FIQDBG_SCHED = (1 << 0),
 	FIQDBG_INT   = (1 << 1),
@@ -128,6 +127,16 @@ enum fiq_debug_level {
 	FIQDBG_PORTHUB = (1 << 3),
 };
 
+typedef struct {
+	union {
+		uint32_t slock;
+		struct _tickets {
+			uint16_t owner;
+			uint16_t next;
+		} tickets;
+	};
+} fiq_lock_t;
+
 struct fiq_state;
 
 extern void _fiq_print (enum fiq_debug_level dbg_lvl, volatile struct fiq_state *state, char *fmt, ...);
@@ -324,6 +333,7 @@ struct fiq_channel_state {
  * It contains top-level state information.
  */
 struct fiq_state {
+	fiq_lock_t lock;
 	mphi_regs_t mphi_regs;
 	void *dwc_regs_base;
 	dma_addr_t dma_base;
@@ -342,6 +352,10 @@ struct fiq_state {
 	struct fiq_channel_state channel[0];
 };
 
+extern void fiq_fsm_spin_lock(fiq_lock_t *lock);
+
+extern void fiq_fsm_spin_unlock(fiq_lock_t *lock);
+
 extern int fiq_fsm_too_late(struct fiq_state *st, int n);
 
 extern int fiq_fsm_tt_in_use(struct fiq_state *st, int num_channels, int n);
--- a/drivers/usb/host/dwc_otg/dwc_otg_hcd.c
+++ b/drivers/usb/host/dwc_otg/dwc_otg_hcd.c
@@ -1184,6 +1184,9 @@ static void assign_and_init_hc(dwc_otg_h
 	dwc_otg_qtd_t *qtd;
 	dwc_otg_hcd_urb_t *urb;
 	void* ptr = NULL;
+	uint32_t intr_enable;
+	unsigned long flags;
+	gintmsk_data_t gintmsk = { .d32 = 0, };
 
 	qtd = DWC_CIRCLEQ_FIRST(&qh->qtd_list);
 
@@ -1409,6 +1412,20 @@ static void assign_and_init_hc(dwc_otg_h
 		hc->desc_list_addr = qh->desc_list_dma;
 
 	dwc_otg_hc_init(hcd->core_if, hc);
+
+	local_irq_save(flags);
+	local_fiq_disable();
+	fiq_fsm_spin_lock(&hcd->fiq_state->lock);
+	/* Enable the top level host channel interrupt. */
+	intr_enable = (1 << hc->hc_num);
+	DWC_MODIFY_REG32(&hcd->core_if->host_if->host_global_regs->haintmsk, 0, intr_enable);
+
+	/* Make sure host channel interrupts are enabled. */
+	gintmsk.b.hcintr = 1;
+	DWC_MODIFY_REG32(&hcd->core_if->core_global_regs->gintmsk, 0, gintmsk.d32);
+	fiq_fsm_spin_unlock(&hcd->fiq_state->lock);
+	local_fiq_enable();
+	local_irq_restore(flags);
 	hc->qh = qh;
 }
 
@@ -1659,6 +1676,7 @@ int fiq_fsm_queue_isoc_transaction(dwc_o
 	fiq_print(FIQDBG_INT, hcd->fiq_state, "%08x", st->hcdma_copy.d32);
 	hfnum.d32 = DWC_READ_REG32(&hcd->core_if->host_if->host_global_regs->hfnum);
 	local_fiq_disable();
+	fiq_fsm_spin_lock(&hcd->fiq_state->lock);
 	DWC_WRITE_REG32(&hc_regs->hctsiz, st->hctsiz_copy.d32);
 	DWC_WRITE_REG32(&hc_regs->hcsplt, st->hcsplt_copy.d32);
 	DWC_WRITE_REG32(&hc_regs->hcdma, st->hcdma_copy.d32);
@@ -1676,6 +1694,7 @@ int fiq_fsm_queue_isoc_transaction(dwc_o
 	}
 	mb();
 	st->hcchar_copy.b.chen = 0;
+	fiq_fsm_spin_unlock(&hcd->fiq_state->lock);
 	local_fiq_enable();
 	return 0;
 }
@@ -1811,7 +1830,7 @@ int fiq_fsm_queue_split_transaction(dwc_
 	DWC_WRITE_REG32(&hc_regs->hcintmsk, st->hcintmsk_copy.d32);
 
 	local_fiq_disable();
-	mb();
+	fiq_fsm_spin_lock(&hcd->fiq_state->lock);
 
 	if (hc->ep_type & 0x1) {
 		hfnum.d32 = DWC_READ_REG32(&hcd->core_if->host_if->host_global_regs->hfnum);
@@ -1909,7 +1928,7 @@ int fiq_fsm_queue_split_transaction(dwc_
 		st->hcchar_copy.b.chen = 1;
 		DWC_WRITE_REG32(&hc_regs->hcchar, st->hcchar_copy.d32);
 	}
-	mb();
+	fiq_fsm_spin_unlock(&hcd->fiq_state->lock);
 	local_fiq_enable();
 	return 0;
 }
--- a/drivers/usb/host/dwc_otg/dwc_otg_hcd_intr.c
+++ b/drivers/usb/host/dwc_otg/dwc_otg_hcd_intr.c
@@ -101,6 +101,7 @@ int32_t dwc_otg_hcd_handle_intr(dwc_otg_
 	if (dwc_otg_is_host_mode(core_if)) {
 		if (fiq_enable) {
 			local_fiq_disable();
+			fiq_fsm_spin_lock(&dwc_otg_hcd->fiq_state->lock);
 			/* Pull in from the FIQ's disabled mask */
 			gintmsk.d32 = gintmsk.d32 | ~(dwc_otg_hcd->fiq_state->gintmsk_saved.d32);
 			dwc_otg_hcd->fiq_state->gintmsk_saved.d32 = ~0;
@@ -116,8 +117,10 @@ int32_t dwc_otg_hcd_handle_intr(dwc_otg_
 		}
 		gintsts.d32 &= gintmsk.d32;
 
-		if (fiq_enable)
+		if (fiq_enable) {
+			fiq_fsm_spin_unlock(&dwc_otg_hcd->fiq_state->lock);
 			local_fiq_enable();
+		}
 
 		if (!gintsts.d32) {
 			goto exit_handler_routine;
@@ -200,6 +203,7 @@ exit_handler_routine:
 		gintmsk_data_t gintmsk_new;
 		haintmsk_data_t haintmsk_new;
 		local_fiq_disable();
+		fiq_fsm_spin_lock(&dwc_otg_hcd->fiq_state->lock);
 		gintmsk_new.d32 = *(volatile uint32_t *)&dwc_otg_hcd->fiq_state->gintmsk_saved.d32;
 		if(fiq_fsm_enable)
 			haintmsk_new.d32 = *(volatile uint32_t *)&dwc_otg_hcd->fiq_state->haintmsk_saved.d32;
@@ -222,6 +226,7 @@ exit_handler_routine:
 		haintmsk.d32 = DWC_READ_REG32(&core_if->host_if->host_global_regs->haintmsk);
 		/* Re-enable interrupts that the FIQ masked (first time round) */
 		FIQ_WRITE(dwc_otg_hcd->fiq_state->dwc_regs_base + GINTMSK, gintmsk.d32);
+		fiq_fsm_spin_unlock(&dwc_otg_hcd->fiq_state->lock);
 		local_fiq_enable();
 
 		if ((jiffies / HZ) > last_time) {
@@ -633,8 +638,10 @@ int32_t dwc_otg_hcd_handle_hc_intr(dwc_o
 	{
 		/* check the mask? */
 		local_fiq_disable();
+		fiq_fsm_spin_lock(&dwc_otg_hcd->fiq_state->lock);
 		haint.b2.chint |= ~(dwc_otg_hcd->fiq_state->haintmsk_saved.b2.chint);
 		dwc_otg_hcd->fiq_state->haintmsk_saved.b2.chint = ~0;
+		fiq_fsm_spin_unlock(&dwc_otg_hcd->fiq_state->lock);
 		local_fiq_enable();
 	}