projects / openwrt / staging / jogo.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
bcm27xx: add linux 5.4 support
[openwrt/staging/jogo.git] / target / linux / bcm27xx / patches-5.4 / 950-0276-bcm2835-dma-Add-proper-40-bit-DMA-support.patch
1 From 773a2db89ad2785d72b215673d87c0a51d769f61 Mon Sep 17 00:00:00 2001
2 From: Phil Elwell <phil@raspberrypi.org>
3 Date: Thu, 4 Apr 2019 13:33:47 +0100
4 Subject: [PATCH] bcm2835-dma: Add proper 40-bit DMA support
5
6 The 40-bit additions are not fully tested, but it should be
7 capable of supporting both 40-bit memcpy on BCM2711 and regular
8 Lite channels on BCM2835.
9
10 Signed-off-by: Phil Elwell <phil@raspberrypi.org>
11 ---
12 drivers/dma/bcm2835-dma.c | 421 ++++++++++++++-----
13 drivers/pci/controller/pcie-brcmstb-bounce.c | 30 +-
14 drivers/pci/controller/pcie-brcmstb-bounce.h | 21 +-
15 drivers/pci/controller/pcie-brcmstb.c | 23 +-
16 4 files changed, 369 insertions(+), 126 deletions(-)
17
18 --- a/drivers/dma/bcm2835-dma.c
19 +++ b/drivers/dma/bcm2835-dma.c
20 @@ -38,6 +38,11 @@
21 #define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14
22 #define BCM2835_DMA_CHAN_NAME_SIZE 8
23 #define BCM2835_DMA_BULK_MASK BIT(0)
24 +#define BCM2838_DMA_MEMCPY_CHAN 14
25 +
26 +struct bcm2835_dma_cfg_data {
27 + u32 chan_40bit_mask;
28 +};
29
30 /**
31 * struct bcm2835_dmadev - BCM2835 DMA controller
32 @@ -52,6 +57,7 @@ struct bcm2835_dmadev {
33 void __iomem *base;
34 struct device_dma_parameters dma_parms;
35 dma_addr_t zero_page;
36 + const struct bcm2835_dma_cfg_data *cfg_data;
37 };
38
39 struct bcm2835_dma_cb {
40 @@ -95,6 +101,7 @@ struct bcm2835_chan {
41 unsigned int irq_flags;
42
43 bool is_lite_channel;
44 + bool is_40bit_channel;
45 };
46
47 struct bcm2835_desc {
48 @@ -184,7 +191,8 @@ struct bcm2835_desc {
49 #define BCM2835_DMA_DATA_TYPE_S128 16
50
51 /* Valid only for channels 0 - 14, 15 has its own base address */
52 -#define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */
53 +#define BCM2835_DMA_CHAN_SIZE 0x100
54 +#define BCM2835_DMA_CHAN(n) ((n) * BCM2835_DMA_CHAN_SIZE) /* Base address */
55 #define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n))
56
57 /* the max dma length for different channels */
58 @@ -195,7 +203,7 @@ struct bcm2835_desc {
59 #define BCM2838_DMA40_CS 0x00
60 #define BCM2838_DMA40_CB 0x04
61 #define BCM2838_DMA40_DEBUG 0x0c
62 -#define BCM2858_DMA40_TI 0x10
63 +#define BCM2838_DMA40_TI 0x10
64 #define BCM2838_DMA40_SRC 0x14
65 #define BCM2838_DMA40_SRCI 0x18
66 #define BCM2838_DMA40_DEST 0x1c
67 @@ -204,32 +212,97 @@ struct bcm2835_desc {
68 #define BCM2838_DMA40_NEXT_CB 0x28
69 #define BCM2838_DMA40_DEBUG2 0x2c
70
71 -#define BCM2838_DMA40_CS_ACTIVE BIT(0)
72 -#define BCM2838_DMA40_CS_END BIT(1)
73 +#define BCM2838_DMA40_ACTIVE BIT(0)
74 +#define BCM2838_DMA40_END BIT(1)
75 +#define BCM2838_DMA40_INT BIT(2)
76 +#define BCM2838_DMA40_DREQ BIT(3) /* DREQ state */
77 +#define BCM2838_DMA40_RD_PAUSED BIT(4) /* Reading is paused */
78 +#define BCM2838_DMA40_WR_PAUSED BIT(5) /* Writing is paused */
79 +#define BCM2838_DMA40_DREQ_PAUSED BIT(6) /* Is paused by DREQ flow control */
80 +#define BCM2838_DMA40_WAITING_FOR_WRITES BIT(7) /* Waiting for last write */
81 +#define BCM2838_DMA40_ERR BIT(10)
82 +#define BCM2838_DMA40_QOS(x) (((x) & 0x1f) << 16)
83 +#define BCM2838_DMA40_PANIC_QOS(x) (((x) & 0x1f) << 20)
84 +#define BCM2838_DMA40_WAIT_FOR_WRITES BIT(28)
85 +#define BCM2838_DMA40_DISDEBUG BIT(29)
86 +#define BCM2838_DMA40_ABORT BIT(30)
87 +#define BCM2838_DMA40_HALT BIT(31)
88 +#define BCM2838_DMA40_CS_FLAGS(x) (x & (BCM2838_DMA40_QOS(15) | \
89 + BCM2838_DMA40_PANIC_QOS(15) | \
90 + BCM2838_DMA40_WAIT_FOR_WRITES | \
91 + BCM2838_DMA40_DISDEBUG))
92 +
93 +/* Transfer information bits */
94 +#define BCM2838_DMA40_INTEN BIT(0)
95 +#define BCM2838_DMA40_TDMODE BIT(1) /* 2D-Mode */
96 +#define BCM2838_DMA40_WAIT_RESP BIT(2) /* wait for AXI write to be acked */
97 +#define BCM2838_DMA40_WAIT_RD_RESP BIT(3) /* wait for AXI read to complete */
98 +#define BCM2838_DMA40_PER_MAP(x) ((x & 31) << 9) /* REQ source */
99 +#define BCM2838_DMA40_S_DREQ BIT(14) /* enable SREQ for source */
100 +#define BCM2838_DMA40_D_DREQ BIT(15) /* enable DREQ for destination */
101 +#define BCM2838_DMA40_S_WAIT(x) ((x & 0xff) << 16) /* add DMA read-wait cycles */
102 +#define BCM2838_DMA40_D_WAIT(x) ((x & 0xff) << 24) /* add DMA write-wait cycles */
103
104 -#define BCM2838_DMA40_CS_QOS(x) (((x) & 0x1f) << 16)
105 -#define BCM2838_DMA40_CS_PANIC_QOS(x) (((x) & 0x1f) << 20)
106 -#define BCM2838_DMA40_CS_WRITE_WAIT BIT(28)
107 +/* debug register bits */
108 +#define BCM2838_DMA40_DEBUG_WRITE_ERR BIT(0)
109 +#define BCM2838_DMA40_DEBUG_FIFO_ERR BIT(1)
110 +#define BCM2838_DMA40_DEBUG_READ_ERR BIT(2)
111 +#define BCM2838_DMA40_DEBUG_READ_CB_ERR BIT(3)
112 +#define BCM2838_DMA40_DEBUG_IN_ON_ERR BIT(8)
113 +#define BCM2838_DMA40_DEBUG_ABORT_ON_ERR BIT(9)
114 +#define BCM2838_DMA40_DEBUG_HALT_ON_ERR BIT(10)
115 +#define BCM2838_DMA40_DEBUG_DISABLE_CLK_GATE BIT(11)
116 +#define BCM2838_DMA40_DEBUG_RSTATE_SHIFT 14
117 +#define BCM2838_DMA40_DEBUG_RSTATE_BITS 4
118 +#define BCM2838_DMA40_DEBUG_WSTATE_SHIFT 18
119 +#define BCM2838_DMA40_DEBUG_WSTATE_BITS 4
120 +#define BCM2838_DMA40_DEBUG_RESET BIT(23)
121 +#define BCM2838_DMA40_DEBUG_ID_SHIFT 24
122 +#define BCM2838_DMA40_DEBUG_ID_BITS 4
123 +#define BCM2838_DMA40_DEBUG_VERSION_SHIFT 28
124 +#define BCM2838_DMA40_DEBUG_VERSION_BITS 4
125 +
126 +/* Valid only for channels 0 - 3 (11 - 14) */
127 +#define BCM2838_DMA40_CHAN(n) (((n) + 11) << 8) /* Base address */
128 +#define BCM2838_DMA40_CHANIO(base, n) ((base) + BCM2838_DMA_CHAN(n))
129
130 -#define BCM2838_DMA40_BURST_LEN(x) ((((x) - 1) & 0xf) << 8)
131 -#define BCM2838_DMA40_INC BIT(12)
132 -#define BCM2838_DMA40_SIZE_128 (2 << 13)
133 +/* the max dma length for different channels */
134 +#define MAX_DMA40_LEN SZ_1G
135
136 -#define BCM2838_DMA40_MEMCPY_QOS \
137 - (BCM2838_DMA40_CS_QOS(0x0) | \
138 - BCM2838_DMA40_CS_PANIC_QOS(0x0) | \
139 - BCM2838_DMA40_CS_WRITE_WAIT)
140 +#define BCM2838_DMA40_BURST_LEN(x) ((min(x,16) - 1) << 8)
141 +#define BCM2838_DMA40_INC BIT(12)
142 +#define BCM2838_DMA40_SIZE_32 (0 << 13)
143 +#define BCM2838_DMA40_SIZE_64 (1 << 13)
144 +#define BCM2838_DMA40_SIZE_128 (2 << 13)
145 +#define BCM2838_DMA40_SIZE_256 (3 << 13)
146 +#define BCM2838_DMA40_IGNORE BIT(15)
147 +#define BCM2838_DMA40_STRIDE(x) ((x) << 16) /* For 2D mode */
148 +
149 +#define BCM2838_DMA40_MEMCPY_FLAGS \
150 + (BCM2838_DMA40_QOS(0) | \
151 + BCM2838_DMA40_PANIC_QOS(0) | \
152 + BCM2838_DMA40_WAIT_FOR_WRITES | \
153 + BCM2838_DMA40_DISDEBUG)
154
155 #define BCM2838_DMA40_MEMCPY_XFER_INFO \
156 (BCM2838_DMA40_SIZE_128 | \
157 BCM2838_DMA40_INC | \
158 BCM2838_DMA40_BURST_LEN(16))
159
160 +struct bcm2835_dmadev *memcpy_parent;
161 static void __iomem *memcpy_chan;
162 static struct bcm2838_dma40_scb *memcpy_scb;
163 static dma_addr_t memcpy_scb_dma;
164 DEFINE_SPINLOCK(memcpy_lock);
165
166 +static const struct bcm2835_dma_cfg_data bcm2835_dma_cfg = {
167 + .chan_40bit_mask = 0,
168 +};
169 +
170 +static const struct bcm2835_dma_cfg_data bcm2838_dma_cfg = {
171 + .chan_40bit_mask = BIT(11) | BIT(12) | BIT(13) | BIT(14),
172 +};
173 +
174 static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c)
175 {
176 /* lite and normal channels have different max frame length */
177 @@ -259,6 +332,32 @@ static inline struct bcm2835_desc *to_bc
178 return container_of(t, struct bcm2835_desc, vd.tx);
179 }
180
181 +static inline uint32_t to_bcm2838_ti(uint32_t info)
182 +{
183 + return ((info & BCM2835_DMA_INT_EN) ? BCM2838_DMA40_INTEN : 0) |
184 + ((info & BCM2835_DMA_WAIT_RESP) ? BCM2838_DMA40_WAIT_RESP : 0) |
185 + ((info & BCM2835_DMA_S_DREQ) ?
186 + (BCM2838_DMA40_S_DREQ | BCM2838_DMA40_WAIT_RD_RESP) : 0) |
187 + ((info & BCM2835_DMA_D_DREQ) ? BCM2838_DMA40_D_DREQ : 0) |
188 + BCM2838_DMA40_PER_MAP((info >> 16) & 0x1f);
189 +}
190 +
191 +static inline uint32_t to_bcm2838_srci(uint32_t info)
192 +{
193 + return ((info & BCM2835_DMA_S_INC) ? BCM2838_DMA40_INC : 0);
194 +}
195 +
196 +static inline uint32_t to_bcm2838_dsti(uint32_t info)
197 +{
198 + return ((info & BCM2835_DMA_D_INC) ? BCM2838_DMA40_INC : 0);
199 +}
200 +
201 +static inline uint32_t to_bcm2838_cbaddr(dma_addr_t addr)
202 +{
203 + BUG_ON(addr & 0x1f);
204 + return (addr >> 5);
205 +}
206 +
207 static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc)
208 {
209 size_t i;
210 @@ -277,45 +376,53 @@ static void bcm2835_dma_desc_free(struct
211 }
212
213 static void bcm2835_dma_create_cb_set_length(
214 - struct bcm2835_chan *chan,
215 + struct bcm2835_chan *c,
216 struct bcm2835_dma_cb *control_block,
217 size_t len,
218 size_t period_len,
219 size_t *total_len,
220 u32 finalextrainfo)
221 {
222 - size_t max_len = bcm2835_dma_max_frame_length(chan);
223 + size_t max_len = bcm2835_dma_max_frame_length(c);
224 + uint32_t cb_len;
225
226 /* set the length taking lite-channel limitations into account */
227 - control_block->length = min_t(u32, len, max_len);
228 + cb_len = min_t(u32, len, max_len);
229
230 - /* finished if we have no period_length */
231 - if (!period_len)
232 - return;
233 + if (period_len) {
234 + /*
235 + * period_len means: that we need to generate
236 + * transfers that are terminating at every
237 + * multiple of period_len - this is typically
238 + * used to set the interrupt flag in info
239 + * which is required during cyclic transfers
240 + */
241
242 - /*
243 - * period_len means: that we need to generate
244 - * transfers that are terminating at every
245 - * multiple of period_len - this is typically
246 - * used to set the interrupt flag in info
247 - * which is required during cyclic transfers
248 - */
249 + /* have we filled in period_length yet? */
250 + if (*total_len + cb_len < period_len) {
251 + /* update number of bytes in this period so far */
252 + *total_len += cb_len;
253 + } else {
254 + /* calculate the length that remains to reach period_len */
255 + cb_len = period_len - *total_len;
256
257 - /* have we filled in period_length yet? */
258 - if (*total_len + control_block->length < period_len) {
259 - /* update number of bytes in this period so far */
260 - *total_len += control_block->length;
261 - return;
262 + /* reset total_length for next period */
263 + *total_len = 0;
264 + }
265 }
266
267 - /* calculate the length that remains to reach period_length */
268 - control_block->length = period_len - *total_len;
269 -
270 - /* reset total_length for next period */
271 - *total_len = 0;
272 -
273 - /* add extrainfo bits in info */
274 - control_block->info |= finalextrainfo;
275 + if (c->is_40bit_channel) {
276 + struct bcm2838_dma40_scb *scb =
277 + (struct bcm2838_dma40_scb *)control_block;
278 +
279 + scb->len = cb_len;
280 + /* add extrainfo bits to ti */
281 + scb->ti |= to_bcm2838_ti(finalextrainfo);
282 + } else {
283 + control_block->length = cb_len;
284 + /* add extrainfo bits to info */
285 + control_block->info |= finalextrainfo;
286 + }
287 }
288
289 static inline size_t bcm2835_dma_count_frames_for_sg(
290 @@ -338,7 +445,7 @@ static inline size_t bcm2835_dma_count_f
291 /**
292 * bcm2835_dma_create_cb_chain - create a control block and fills data in
293 *
294 - * @chan: the @dma_chan for which we run this
295 + * @c: the @bcm2835_chan for which we run this
296 * @direction: the direction in which we transfer
297 * @cyclic: it is a cyclic transfer
298 * @info: the default info bits to apply per controlblock
299 @@ -356,12 +463,11 @@ static inline size_t bcm2835_dma_count_f
300 * @gfp: the GFP flag to use for allocation
301 */
302 static struct bcm2835_desc *bcm2835_dma_create_cb_chain(
303 - struct dma_chan *chan, enum dma_transfer_direction direction,
304 + struct bcm2835_chan *c, enum dma_transfer_direction direction,
305 bool cyclic, u32 info, u32 finalextrainfo, size_t frames,
306 dma_addr_t src, dma_addr_t dst, size_t buf_len,
307 size_t period_len, gfp_t gfp)
308 {
309 - struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
310 size_t len = buf_len, total_len;
311 size_t frame;
312 struct bcm2835_desc *d;
313 @@ -393,11 +499,23 @@ static struct bcm2835_desc *bcm2835_dma_
314
315 /* fill in the control block */
316 control_block = cb_entry->cb;
317 - control_block->info = info;
318 - control_block->src = src;
319 - control_block->dst = dst;
320 - control_block->stride = 0;
321 - control_block->next = 0;
322 + if (c->is_40bit_channel) {
323 + struct bcm2838_dma40_scb *scb =
324 + (struct bcm2838_dma40_scb *)control_block;
325 + scb->ti = to_bcm2838_ti(info);
326 + scb->src = lower_32_bits(src);
327 + scb->srci= upper_32_bits(src) | to_bcm2838_srci(info);
328 + scb->dst = lower_32_bits(dst);
329 + scb->dsti = upper_32_bits(dst) | to_bcm2838_dsti(info);
330 + scb->next_cb = 0;
331 + } else {
332 + control_block->info = info;
333 + control_block->src = src;
334 + control_block->dst = dst;
335 + control_block->stride = 0;
336 + control_block->next = 0;
337 + }
338 +
339 /* set up length in control_block if requested */
340 if (buf_len) {
341 /* calculate length honoring period_length */
342 @@ -411,7 +529,10 @@ static struct bcm2835_desc *bcm2835_dma_
343 }
344
345 /* link this the last controlblock */
346 - if (frame)
347 + if (frame && c->is_40bit_channel)
348 + d->cb_list[frame - 1].cb->next =
349 + to_bcm2838_cbaddr(cb_entry->paddr);
350 + if (frame && !c->is_40bit_channel)
351 d->cb_list[frame - 1].cb->next = cb_entry->paddr;
352
353 /* update src and dst and length */
354 @@ -425,7 +546,14 @@ static struct bcm2835_desc *bcm2835_dma_
355 }
356
357 /* the last frame requires extra flags */
358 - d->cb_list[d->frames - 1].cb->info |= finalextrainfo;
359 + if (c->is_40bit_channel) {
360 + struct bcm2838_dma40_scb *scb =
361 + (struct bcm2838_dma40_scb *)d->cb_list[d->frames-1].cb;
362 +
363 + scb->ti |= to_bcm2838_ti(finalextrainfo);
364 + } else {
365 + d->cb_list[d->frames - 1].cb->info |= finalextrainfo;
366 + }
367
368 /* detect a size missmatch */
369 if (buf_len && (d->size != buf_len))
370 @@ -439,13 +567,12 @@ error_cb:
371 }
372
373 static void bcm2835_dma_fill_cb_chain_with_sg(
374 - struct dma_chan *chan,
375 + struct bcm2835_chan *c,
376 enum dma_transfer_direction direction,
377 struct bcm2835_cb_entry *cb,
378 struct scatterlist *sgl,
379 unsigned int sg_len)
380 {
381 - struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
382 size_t len, max_len;
383 unsigned int i;
384 dma_addr_t addr;
385 @@ -453,14 +580,34 @@ static void bcm2835_dma_fill_cb_chain_wi
386
387 max_len = bcm2835_dma_max_frame_length(c);
388 for_each_sg(sgl, sgent, sg_len, i) {
389 - for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent);
390 - len > 0;
391 - addr += cb->cb->length, len -= cb->cb->length, cb++) {
392 - if (direction == DMA_DEV_TO_MEM)
393 - cb->cb->dst = addr;
394 - else
395 - cb->cb->src = addr;
396 - cb->cb->length = min(len, max_len);
397 + if (c->is_40bit_channel) {
398 + struct bcm2838_dma40_scb *scb =
399 + (struct bcm2838_dma40_scb *)cb->cb;
400 + for (addr = sg_dma_address(sgent),
401 + len = sg_dma_len(sgent);
402 + len > 0;
403 + addr += scb->len, len -= scb->len, scb++) {
404 + if (direction == DMA_DEV_TO_MEM) {
405 + scb->dst = lower_32_bits(addr);
406 + scb->dsti = upper_32_bits(addr) | BCM2838_DMA40_INC;
407 + } else {
408 + scb->src = lower_32_bits(addr);
409 + scb->srci = upper_32_bits(addr) | BCM2838_DMA40_INC;
410 + }
411 + scb->len = min(len, max_len);
412 + }
413 + } else {
414 + for (addr = sg_dma_address(sgent),
415 + len = sg_dma_len(sgent);
416 + len > 0;
417 + addr += cb->cb->length, len -= cb->cb->length,
418 + cb++) {
419 + if (direction == DMA_DEV_TO_MEM)
420 + cb->cb->dst = addr;
421 + else
422 + cb->cb->src = addr;
423 + cb->cb->length = min(len, max_len);
424 + }
425 }
426 }
427 }
428 @@ -469,6 +616,10 @@ static void bcm2835_dma_abort(struct bcm
429 {
430 void __iomem *chan_base = c->chan_base;
431 long int timeout = 10000;
432 + u32 wait_mask = BCM2835_DMA_WAITING_FOR_WRITES;
433 +
434 + if (c->is_40bit_channel)
435 + wait_mask = BCM2838_DMA40_WAITING_FOR_WRITES;
436
437 /*
438 * A zero control block address means the channel is idle.
439 @@ -481,8 +632,7 @@ static void bcm2835_dma_abort(struct bcm
440 writel(0, chan_base + BCM2835_DMA_CS);
441
442 /* Wait for any current AXI transfer to complete */
443 - while ((readl(chan_base + BCM2835_DMA_CS) &
444 - BCM2835_DMA_WAITING_FOR_WRITES) && --timeout)
445 + while ((readl(chan_base + BCM2835_DMA_CS) & wait_mask) && --timeout)
446 cpu_relax();
447
448 /* Peripheral might be stuck and fail to signal AXI write responses */
449 @@ -507,9 +657,16 @@ static void bcm2835_dma_start_desc(struc
450
451 c->desc = d = to_bcm2835_dma_desc(&vd->tx);
452
453 - writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);
454 - writel(BCM2835_DMA_ACTIVE | BCM2835_DMA_CS_FLAGS(c->dreq),
455 - c->chan_base + BCM2835_DMA_CS);
456 + if (c->is_40bit_channel) {
457 + writel(to_bcm2838_cbaddr(d->cb_list[0].paddr),
458 + c->chan_base + BCM2838_DMA40_CB);
459 + writel(BCM2838_DMA40_ACTIVE | BCM2838_DMA40_CS_FLAGS(c->dreq),
460 + c->chan_base + BCM2838_DMA40_CS);
461 + } else {
462 + writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);
463 + writel(BCM2835_DMA_ACTIVE | BCM2835_DMA_CS_FLAGS(c->dreq),
464 + c->chan_base + BCM2835_DMA_CS);
465 + }
466 }
467
468 static irqreturn_t bcm2835_dma_callback(int irq, void *data)
469 @@ -537,7 +694,8 @@ static irqreturn_t bcm2835_dma_callback(
470 * will remain idle despite the ACTIVE flag being set.
471 */
472 writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE |
473 - BCM2835_DMA_CS_FLAGS(c->dreq),
474 + (c->is_40bit_channel ? BCM2838_DMA40_CS_FLAGS(c->dreq) :
475 + BCM2835_DMA_CS_FLAGS(c->dreq)),
476 c->chan_base + BCM2835_DMA_CS);
477
478 d = c->desc;
479 @@ -640,9 +798,17 @@ static enum dma_status bcm2835_dma_tx_st
480 struct bcm2835_desc *d = c->desc;
481 dma_addr_t pos;
482
483 - if (d->dir == DMA_MEM_TO_DEV)
484 + if (d->dir == DMA_MEM_TO_DEV && c->is_40bit_channel)
485 + pos = readl(c->chan_base + BCM2838_DMA40_SRC) +
486 + ((readl(c->chan_base + BCM2838_DMA40_SRCI) &
487 + 0xff) << 8);
488 + else if (d->dir == DMA_MEM_TO_DEV && !c->is_40bit_channel)
489 pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD);
490 - else if (d->dir == DMA_DEV_TO_MEM)
491 + else if (d->dir == DMA_DEV_TO_MEM && c->is_40bit_channel)
492 + pos = readl(c->chan_base + BCM2838_DMA40_DEST) +
493 + ((readl(c->chan_base + BCM2838_DMA40_DESTI) &
494 + 0xff) << 8);
495 + else if (d->dir == DMA_DEV_TO_MEM && !c->is_40bit_channel)
496 pos = readl(c->chan_base + BCM2835_DMA_DEST_AD);
497 else
498 pos = 0;
499 @@ -688,7 +854,7 @@ static struct dma_async_tx_descriptor *b
500 frames = bcm2835_dma_frames_for_length(len, max_len);
501
502 /* allocate the CB chain - this also fills in the pointers */
503 - d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false,
504 + d = bcm2835_dma_create_cb_chain(c, DMA_MEM_TO_MEM, false,
505 info, extra, frames,
506 src, dst, len, 0, GFP_KERNEL);
507 if (!d)
508 @@ -723,11 +889,21 @@ static struct dma_async_tx_descriptor *b
509 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
510 return NULL;
511 src = c->cfg.src_addr;
512 + /*
513 + * One would think it ought to be possible to get the physical
514 + * to dma address mapping information from the dma-ranges DT
515 + * property, but I've not found a way yet that doesn't involve
516 + * open-coding the whole thing.
517 + */
518 + if (c->is_40bit_channel)
519 + src |= 0x400000000ull;
520 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
521 } else {
522 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
523 return NULL;
524 dst = c->cfg.dst_addr;
525 + if (c->is_40bit_channel)
526 + dst |= 0x400000000ull;
527 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
528 }
529
530 @@ -735,7 +911,7 @@ static struct dma_async_tx_descriptor *b
531 frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len);
532
533 /* allocate the CB chain */
534 - d = bcm2835_dma_create_cb_chain(chan, direction, false,
535 + d = bcm2835_dma_create_cb_chain(c, direction, false,
536 info, extra,
537 frames, src, dst, 0, 0,
538 GFP_NOWAIT);
539 @@ -743,7 +919,7 @@ static struct dma_async_tx_descriptor *b
540 return NULL;
541
542 /* fill in frames with scatterlist pointers */
543 - bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list,
544 + bcm2835_dma_fill_cb_chain_with_sg(c, direction, d->cb_list,
545 sgl, sg_len);
546
547 return vchan_tx_prep(&c->vc, &d->vd, flags);
548 @@ -822,7 +998,7 @@ static struct dma_async_tx_descriptor *b
549 * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine
550 * implementation calls prep_dma_cyclic with interrupts disabled.
551 */
552 - d = bcm2835_dma_create_cb_chain(chan, direction, true,
553 + d = bcm2835_dma_create_cb_chain(c, direction, true,
554 info, extra,
555 frames, src, dst, buf_len,
556 period_len, GFP_NOWAIT);
557 @@ -830,7 +1006,8 @@ static struct dma_async_tx_descriptor *b
558 return NULL;
559
560 /* wrap around into a loop */
561 - d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr;
562 + d->cb_list[d->frames - 1].cb->next = c->is_40bit_channel ?
563 + to_bcm2838_cbaddr(d->cb_list[0].paddr) : d->cb_list[0].paddr;
564
565 return vchan_tx_prep(&c->vc, &d->vd, flags);
566 }
567 @@ -894,9 +1071,11 @@ static int bcm2835_dma_chan_init(struct
568 c->irq_number = irq;
569 c->irq_flags = irq_flags;
570
571 - /* check in DEBUG register if this is a LITE channel */
572 - if (readl(c->chan_base + BCM2835_DMA_DEBUG) &
573 - BCM2835_DMA_DEBUG_LITE)
574 + /* check for 40bit and lite channels */
575 + if (d->cfg_data->chan_40bit_mask & BIT(chan_id))
576 + c->is_40bit_channel = true;
577 + else if (readl(c->chan_base + BCM2835_DMA_DEBUG) &
578 + BCM2835_DMA_DEBUG_LITE)
579 c->is_lite_channel = true;
580
581 return 0;
582 @@ -916,18 +1095,16 @@ static void bcm2835_dma_free(struct bcm2
583 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
584 }
585
586 -int bcm2838_dma40_memcpy_init(struct device *dev)
587 +int bcm2838_dma40_memcpy_init(void)
588 {
589 - if (memcpy_scb)
590 - return 0;
591 + if (!memcpy_parent)
592 + return -EPROBE_DEFER;
593
594 - memcpy_scb = dma_alloc_coherent(dev, sizeof(*memcpy_scb),
595 - &memcpy_scb_dma, GFP_KERNEL);
596 + if (!memcpy_chan)
597 + return -EINVAL;
598
599 - if (!memcpy_scb) {
600 - pr_err("bcm2838_dma40_memcpy_init failed!\n");
601 + if (!memcpy_scb)
602 return -ENOMEM;
603 - }
604
605 return 0;
606 }
607 @@ -954,20 +1131,22 @@ void bcm2838_dma40_memcpy(dma_addr_t dst
608 scb->next_cb = 0;
609
610 writel((u32)(memcpy_scb_dma >> 5), memcpy_chan + BCM2838_DMA40_CB);
611 - writel(BCM2838_DMA40_MEMCPY_QOS + BCM2838_DMA40_CS_ACTIVE,
612 + writel(BCM2838_DMA40_MEMCPY_FLAGS + BCM2838_DMA40_ACTIVE,
613 memcpy_chan + BCM2838_DMA40_CS);
614 +
615 /* Poll for completion */
616 - while (!(readl(memcpy_chan + BCM2838_DMA40_CS) & BCM2838_DMA40_CS_END))
617 + while (!(readl(memcpy_chan + BCM2838_DMA40_CS) & BCM2838_DMA40_END))
618 cpu_relax();
619
620 - writel(BCM2838_DMA40_CS_END, memcpy_chan + BCM2838_DMA40_CS);
621 + writel(BCM2838_DMA40_END, memcpy_chan + BCM2838_DMA40_CS);
622
623 spin_unlock_irqrestore(&memcpy_lock, flags);
624 }
625 EXPORT_SYMBOL(bcm2838_dma40_memcpy);
626
627 static const struct of_device_id bcm2835_dma_of_match[] = {
628 - { .compatible = "brcm,bcm2835-dma", },
629 + { .compatible = "brcm,bcm2835-dma", .data = &bcm2835_dma_cfg },
630 + { .compatible = "brcm,bcm2838-dma", .data = &bcm2838_dma_cfg },
631 {},
632 };
633 MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match);
634 @@ -999,6 +1178,8 @@ static int bcm2835_dma_probe(struct plat
635 int irq_flags;
636 uint32_t chans_available;
637 char chan_name[BCM2835_DMA_CHAN_NAME_SIZE];
638 + const struct of_device_id *of_id;
639 + int chan_count, chan_start, chan_end;
640
641 if (!pdev->dev.dma_mask)
642 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
643 @@ -1020,9 +1201,13 @@ static int bcm2835_dma_probe(struct plat
644 base = devm_ioremap_resource(&pdev->dev, res);
645 if (IS_ERR(base))
646 return PTR_ERR(base);
647 - rc = bcm_dmaman_probe(pdev, base, BCM2835_DMA_BULK_MASK);
648 - if (rc)
649 - dev_err(&pdev->dev, "Failed to initialize the legacy API\n");
650 +
651 + /* The set of channels can be split across multiple instances. */
652 + chan_start = ((u32)(uintptr_t)base / BCM2835_DMA_CHAN_SIZE) & 0xf;
653 + base -= BCM2835_DMA_CHAN(chan_start);
654 + chan_count = resource_size(res) / BCM2835_DMA_CHAN_SIZE;
655 + chan_end = min(chan_start + chan_count,
656 + BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1);
657
658 od->base = base;
659
660 @@ -1059,6 +1244,14 @@ static int bcm2835_dma_probe(struct plat
661 return -ENOMEM;
662 }
663
664 + of_id = of_match_node(bcm2835_dma_of_match, pdev->dev.of_node);
665 + if (!of_id) {
666 + dev_err(&pdev->dev, "Failed to match compatible string\n");
667 + return -EINVAL;
668 + }
669 +
670 + od->cfg_data = of_id->data;
671 +
672 /* Request DMA channel mask from device tree */
673 if (of_property_read_u32(pdev->dev.of_node,
674 "brcm,dma-channel-mask",
675 @@ -1068,18 +1261,34 @@ static int bcm2835_dma_probe(struct plat
676 goto err_no_dma;
677 }
678
679 - /* Channel 0 is used by the legacy API */
680 - chans_available &= ~BCM2835_DMA_BULK_MASK;
681 + /* One channel is reserved for the legacy API */
682 + if (chans_available & BCM2835_DMA_BULK_MASK) {
683 + rc = bcm_dmaman_probe(pdev, base,
684 + chans_available & BCM2835_DMA_BULK_MASK);
685 + if (rc)
686 + dev_err(&pdev->dev,
687 + "Failed to initialize the legacy API\n");
688 +
689 + chans_available &= ~BCM2835_DMA_BULK_MASK;
690 + }
691
692 - /* We can't use channels 11-13 yet */
693 - chans_available &= ~(BIT(11) | BIT(12) | BIT(13));
694 + /* And possibly one for the 40-bit DMA memcpy API */
695 + if (chans_available & od->cfg_data->chan_40bit_mask &
696 + BIT(BCM2838_DMA_MEMCPY_CHAN)) {
697 + memcpy_parent = od;
698 + memcpy_chan = BCM2835_DMA_CHANIO(base, BCM2838_DMA_MEMCPY_CHAN);
699 + memcpy_scb = dma_alloc_coherent(memcpy_parent->ddev.dev,
700 + sizeof(*memcpy_scb),
701 + &memcpy_scb_dma, GFP_KERNEL);
702 + if (!memcpy_scb)
703 + dev_warn(&pdev->dev,
704 + "Failed to allocated memcpy scb\n");
705
706 - /* Grab channel 14 for the 40-bit DMA memcpy */
707 - chans_available &= ~BIT(14);
708 - memcpy_chan = BCM2835_DMA_CHANIO(base, 14);
709 + chans_available &= ~BIT(BCM2838_DMA_MEMCPY_CHAN);
710 + }
711
712 /* get irqs for each channel that we support */
713 - for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
714 + for (i = chan_start; i < chan_end; i++) {
715 /* skip masked out channels */
716 if (!(chans_available & (1 << i))) {
717 irq[i] = -1;
718 @@ -1102,13 +1311,17 @@ static int bcm2835_dma_probe(struct plat
719 irq[i] = platform_get_irq(pdev, i < 11 ? i : 11);
720 }
721
722 + chan_count = 0;
723 +
724 /* get irqs for each channel */
725 - for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
726 + for (i = chan_start; i < chan_end; i++) {
727 /* skip channels without irq */
728 if (irq[i] < 0)
729 continue;
730
731 /* check if there are other channels that also use this irq */
732 + /* FIXME: This will fail if interrupts are shared across
733 + instances */
734 irq_flags = 0;
735 for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++)
736 if ((i != j) && (irq[j] == irq[i])) {
737 @@ -1120,9 +1333,10 @@ static int bcm2835_dma_probe(struct plat
738 rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags);
739 if (rc)
740 goto err_no_dma;
741 + chan_count++;
742 }
743
744 - dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i);
745 + dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", chan_count);
746
747 /* Device-tree DMA controller registration */
748 rc = of_dma_controller_register(pdev->dev.of_node,
749 @@ -1154,6 +1368,13 @@ static int bcm2835_dma_remove(struct pla
750
751 bcm_dmaman_remove(pdev);
752 dma_async_device_unregister(&od->ddev);
753 + if (memcpy_parent == od) {
754 + dma_free_coherent(&pdev->dev, sizeof(*memcpy_scb), memcpy_scb,
755 + memcpy_scb_dma);
756 + memcpy_parent = NULL;
757 + memcpy_scb = NULL;
758 + memcpy_chan = NULL;
759 + }
760 bcm2835_dma_free(od);
761
762 return 0;
763 --- a/drivers/pci/controller/pcie-brcmstb-bounce.c
764 +++ b/drivers/pci/controller/pcie-brcmstb-bounce.c
765 @@ -91,7 +91,7 @@ struct dmabounce_device_info {
766
767 static struct dmabounce_device_info *g_dmabounce_device_info;
768
769 -extern int bcm2838_dma40_memcpy_init(struct device *dev);
770 +extern int bcm2838_dma40_memcpy_init(void);
771 extern void bcm2838_dma40_memcpy(dma_addr_t dst, dma_addr_t src, size_t size);
772
773 #ifdef STATS
774 @@ -465,9 +465,9 @@ static const struct dma_map_ops dmabounc
775 .dma_supported = dmabounce_dma_supported,
776 };
777
778 -int brcm_pcie_bounce_register_dev(struct device *dev,
779 - unsigned long buffer_size,
780 - dma_addr_t threshold)
781 +int brcm_pcie_bounce_init(struct device *dev,
782 + unsigned long buffer_size,
783 + dma_addr_t threshold)
784 {
785 struct dmabounce_device_info *device_info;
786 int ret;
787 @@ -476,9 +476,9 @@ int brcm_pcie_bounce_register_dev(struct
788 if (g_dmabounce_device_info)
789 return -EBUSY;
790
791 - ret = bcm2838_dma40_memcpy_init(dev);
792 + ret = bcm2838_dma40_memcpy_init();
793 if (ret)
794 - return ret;
795 + return ret;
796
797 device_info = kmalloc(sizeof(struct dmabounce_device_info), GFP_ATOMIC);
798 if (!device_info) {
799 @@ -509,9 +509,8 @@ int brcm_pcie_bounce_register_dev(struct
800 device_create_file(dev, &dev_attr_dmabounce_stats));
801
802 g_dmabounce_device_info = device_info;
803 - set_dma_ops(dev, &dmabounce_ops);
804
805 - dev_info(dev, "dmabounce: registered device - %ld kB, threshold %pad\n",
806 + dev_info(dev, "dmabounce: initialised - %ld kB, threshold %pad\n",
807 buffer_size / 1024, &threshold);
808
809 return 0;
810 @@ -520,14 +519,13 @@ int brcm_pcie_bounce_register_dev(struct
811 kfree(device_info);
812 return ret;
813 }
814 -EXPORT_SYMBOL(brcm_pcie_bounce_register_dev);
815 +EXPORT_SYMBOL(brcm_pcie_bounce_init);
816
817 -void brcm_pcie_bounce_unregister_dev(struct device *dev)
818 +void brcm_pcie_bounce_uninit(struct device *dev)
819 {
820 struct dmabounce_device_info *device_info = g_dmabounce_device_info;
821
822 g_dmabounce_device_info = NULL;
823 - set_dma_ops(dev, NULL);
824
825 if (!device_info) {
826 dev_warn(dev,
827 @@ -548,10 +546,16 @@ void brcm_pcie_bounce_unregister_dev(str
828 device_remove_file(dev, &dev_attr_dmabounce_stats));
829
830 kfree(device_info);
831 +}
832 +EXPORT_SYMBOL(brcm_pcie_bounce_uninit);
833 +
834 +int brcm_pcie_bounce_register_dev(struct device *dev)
835 +{
836 + set_dma_ops(dev, &dmabounce_ops);
837
838 - dev_info(dev, "dmabounce: device unregistered\n");
839 + return 0;
840 }
841 -EXPORT_SYMBOL(brcm_pcie_bounce_unregister_dev);
842 +EXPORT_SYMBOL(brcm_pcie_bounce_register_dev);
843
844 MODULE_AUTHOR("Phil Elwell <phil@raspberrypi.org>");
845 MODULE_DESCRIPTION("Dedicate DMA bounce support for pcie-brcmstb");
846 --- a/drivers/pci/controller/pcie-brcmstb-bounce.h
847 +++ b/drivers/pci/controller/pcie-brcmstb-bounce.h
848 @@ -8,21 +8,26 @@
849
850 #ifdef CONFIG_ARM
851
852 -int brcm_pcie_bounce_register_dev(struct device *dev, unsigned long buffer_size,
853 - dma_addr_t threshold);
854 -
855 -int brcm_pcie_bounce_unregister_dev(struct device *dev);
856 +int brcm_pcie_bounce_init(struct device *dev, unsigned long buffer_size,
857 + dma_addr_t threshold);
858 +int brcm_pcie_bounce_uninit(struct device *dev);
859 +int brcm_pcie_bounce_register_dev(struct device *dev);
860
861 #else
862
863 -static inline int brcm_pcie_bounce_register_dev(struct device *dev,
864 - unsigned long buffer_size,
865 - dma_addr_t threshold)
866 +static inline int brcm_pcie_bounce_init(struct device *dev,
867 + unsigned long buffer_size,
868 + dma_addr_t threshold)
869 +{
870 + return 0;
871 +}
872 +
873 +static inline int brcm_pcie_bounce_uninit(struct device *dev)
874 {
875 return 0;
876 }
877
878 -static inline int brcm_pcie_bounce_unregister_dev(struct device *dev)
879 +static inline int brcm_pcie_bounce_register_dev(struct device *dev)
880 {
881 return 0;
882 }
883 --- a/drivers/pci/controller/pcie-brcmstb.c
884 +++ b/drivers/pci/controller/pcie-brcmstb.c
885 @@ -644,6 +644,7 @@ static void brcm_set_dma_ops(struct devi
886
887 static inline void brcm_pcie_perst_set(struct brcm_pcie *pcie,
888 unsigned int val);
889 +
890 static int brcmstb_platform_notifier(struct notifier_block *nb,
891 unsigned long event, void *__dev)
892 {
893 @@ -657,12 +658,11 @@ static int brcmstb_platform_notifier(str
894 strcmp(dev->kobj.name, rc_name)) {
895 int ret;
896
897 - ret = brcm_pcie_bounce_register_dev(dev, bounce_buffer,
898 - (dma_addr_t)bounce_threshold);
899 + ret = brcm_pcie_bounce_register_dev(dev);
900 if (ret) {
901 dev_err(dev,
902 "brcm_pcie_bounce_register_dev() failed: %d\n",
903 - ret);
904 + ret);
905 return ret;
906 }
907 }
908 @@ -675,8 +675,6 @@ static int brcmstb_platform_notifier(str
909 brcm_pcie_perst_set(g_pcie, 1);
910 msleep(100);
911 brcm_pcie_perst_set(g_pcie, 0);
912 - } else if (max_pfn > (bounce_threshold/PAGE_SIZE)) {
913 - brcm_pcie_bounce_unregister_dev(dev);
914 }
915 return NOTIFY_OK;
916
917 @@ -1712,6 +1710,7 @@ static int brcm_pcie_probe(struct platfo
918 void __iomem *base;
919 struct pci_host_bridge *bridge;
920 struct pci_bus *child;
921 + extern unsigned long max_pfn;
922
923 bridge = devm_pci_alloc_host_bridge(&pdev->dev, sizeof(*pcie));
924 if (!bridge)
925 @@ -1747,6 +1746,20 @@ static int brcm_pcie_probe(struct platfo
926 if (IS_ERR(base))
927 return PTR_ERR(base);
928
929 + /* To Do: Add hardware check if this ever gets fixed */
930 + if (max_pfn > (bounce_threshold/PAGE_SIZE)) {
931 + int ret;
932 + ret = brcm_pcie_bounce_init(&pdev->dev, bounce_buffer,
933 + (dma_addr_t)bounce_threshold);
934 + if (ret) {
935 + if (ret != -EPROBE_DEFER)
936 + dev_err(&pdev->dev,
937 + "could not init bounce buffers: %d\n",
938 + ret);
939 + return ret;
940 + }
941 + }
942 +
943 pcie->clk = of_clk_get_by_name(dn, "sw_pcie");
944 if (IS_ERR(pcie->clk)) {
945 dev_warn(&pdev->dev, "could not get clock\n");
Staging tree of Jonas Gorski