projects / openwrt / staging / dedeckeh.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
layerscape: refresh patches
[openwrt/staging/dedeckeh.git] / target / linux / layerscape / patches-4.9 / 805-dma-support-layerscape.patch
1 From 659603c5f6cbc3d39922d4374df25ae4627d0e88 Mon Sep 17 00:00:00 2001
2 From: Yangbo Lu <yangbo.lu@nxp.com>
3 Date: Mon, 25 Sep 2017 12:12:20 +0800
4 Subject: [PATCH] dma: support layerscape
5
6 This is a integrated patch for layerscape dma support.
7
8 Signed-off-by: jiaheng.fan <jiaheng.fan@nxp.com>
9 Signed-off-by: Yangbo Lu <yangbo.lu@nxp.com>
10 ---
11 drivers/dma/Kconfig | 14 +
12 drivers/dma/Makefile | 2 +
13 drivers/dma/dpaa2-qdma/Kconfig | 8 +
14 drivers/dma/dpaa2-qdma/Makefile | 8 +
15 drivers/dma/dpaa2-qdma/dpaa2-qdma.c | 986 +++++++++++++++++++++++++
16 drivers/dma/dpaa2-qdma/dpaa2-qdma.h | 262 +++++++
17 drivers/dma/dpaa2-qdma/dpdmai.c | 454 ++++++++++++
18 drivers/dma/dpaa2-qdma/fsl_dpdmai.h | 521 ++++++++++++++
19 drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h | 222 ++++++
20 drivers/dma/fsl-qdma.c | 1201 +++++++++++++++++++++++++++++++
21 10 files changed, 3678 insertions(+)
22 create mode 100644 drivers/dma/dpaa2-qdma/Kconfig
23 create mode 100644 drivers/dma/dpaa2-qdma/Makefile
24 create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.c
25 create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.h
26 create mode 100644 drivers/dma/dpaa2-qdma/dpdmai.c
27 create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai.h
28 create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h
29 create mode 100644 drivers/dma/fsl-qdma.c
30
31 --- a/drivers/dma/Kconfig
32 +++ b/drivers/dma/Kconfig
33 @@ -192,6 +192,20 @@ config FSL_EDMA
34 multiplexing capability for DMA request sources(slot).
35 This module can be found on Freescale Vybrid and LS-1 SoCs.
36
37 +config FSL_QDMA
38 + tristate "Freescale qDMA engine support"
39 + select DMA_ENGINE
40 + select DMA_VIRTUAL_CHANNELS
41 + select DMA_ENGINE_RAID
42 + select ASYNC_TX_ENABLE_CHANNEL_SWITCH
43 + help
44 + Support the Freescale qDMA engine with command queue and legacy mode.
45 + Channel virtualization is supported through enqueuing of DMA jobs to,
46 + or dequeuing DMA jobs from, different work queues.
47 + This module can be found on Freescale LS SoCs.
48 +
49 +source drivers/dma/dpaa2-qdma/Kconfig
50 +
51 config FSL_RAID
52 tristate "Freescale RAID engine Support"
53 depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH
54 --- a/drivers/dma/Makefile
55 +++ b/drivers/dma/Makefile
56 @@ -29,6 +29,8 @@ obj-$(CONFIG_DW_DMAC_CORE) += dw/
57 obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o
58 obj-$(CONFIG_FSL_DMA) += fsldma.o
59 obj-$(CONFIG_FSL_EDMA) += fsl-edma.o
60 +obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o
61 +obj-$(CONFIG_FSL_DPAA2_QDMA) += dpaa2-qdma/
62 obj-$(CONFIG_FSL_RAID) += fsl_raid.o
63 obj-$(CONFIG_HSU_DMA) += hsu/
64 obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o
65 --- /dev/null
66 +++ b/drivers/dma/dpaa2-qdma/Kconfig
67 @@ -0,0 +1,8 @@
68 +menuconfig FSL_DPAA2_QDMA
69 + tristate "NXP DPAA2 QDMA"
70 + depends on FSL_MC_BUS && FSL_MC_DPIO
71 + select DMA_ENGINE
72 + select DMA_VIRTUAL_CHANNELS
73 + ---help---
74 + NXP Data Path Acceleration Architecture 2 QDMA driver,
75 + using the NXP MC bus driver.
76 --- /dev/null
77 +++ b/drivers/dma/dpaa2-qdma/Makefile
78 @@ -0,0 +1,8 @@
79 +#
80 +# Makefile for the NXP DPAA2 CAAM controllers
81 +#
82 +ccflags-y += -DVERSION=\"\"
83 +
84 +obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma.o
85 +
86 +fsl-dpaa2-qdma-objs := dpaa2-qdma.o dpdmai.o
87 --- /dev/null
88 +++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.c
89 @@ -0,0 +1,986 @@
90 +/*
91 + * drivers/dma/dpaa2-qdma/dpaa2-qdma.c
92 + *
93 + * Copyright 2015-2017 NXP Semiconductor, Inc.
94 + * Author: Changming Huang <jerry.huang@nxp.com>
95 + *
96 + * Driver for the NXP QDMA engine with QMan mode.
97 + * Channel virtualization is supported through enqueuing of DMA jobs to,
98 + * or dequeuing DMA jobs from different work queues with QMan portal.
99 + * This module can be found on NXP LS2 SoCs.
100 + *
101 + * This program is free software; you can redistribute it and/or modify it
102 + * under the terms of the GNU General Public License as published by the
103 + * Free Software Foundation; either version 2 of the License, or (at your
104 + * option) any later version.
105 + */
106 +
107 +#include <linux/init.h>
108 +#include <linux/module.h>
109 +#include <linux/interrupt.h>
110 +#include <linux/clk.h>
111 +#include <linux/dma-mapping.h>
112 +#include <linux/dmapool.h>
113 +#include <linux/slab.h>
114 +#include <linux/spinlock.h>
115 +#include <linux/of.h>
116 +#include <linux/of_device.h>
117 +#include <linux/of_address.h>
118 +#include <linux/of_irq.h>
119 +#include <linux/of_dma.h>
120 +#include <linux/types.h>
121 +#include <linux/delay.h>
122 +#include <linux/iommu.h>
123 +
124 +#include "../virt-dma.h"
125 +
126 +#include "../../../drivers/staging/fsl-mc/include/mc.h"
127 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h"
128 +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
129 +#include "fsl_dpdmai_cmd.h"
130 +#include "fsl_dpdmai.h"
131 +#include "dpaa2-qdma.h"
132 +
133 +static bool smmu_disable = true;
134 +
135 +static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan)
136 +{
137 + return container_of(chan, struct dpaa2_qdma_chan, vchan.chan);
138 +}
139 +
140 +static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
141 +{
142 + return container_of(vd, struct dpaa2_qdma_comp, vdesc);
143 +}
144 +
145 +static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan)
146 +{
147 + return 0;
148 +}
149 +
150 +static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan)
151 +{
152 + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
153 + unsigned long flags;
154 + LIST_HEAD(head);
155 +
156 + spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags);
157 + vchan_get_all_descriptors(&dpaa2_chan->vchan, &head);
158 + spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags);
159 +
160 + vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head);
161 +}
162 +
163 +/*
164 + * Request a command descriptor for enqueue.
165 + */
166 +static struct dpaa2_qdma_comp *
167 +dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan)
168 +{
169 + struct dpaa2_qdma_comp *comp_temp = NULL;
170 + unsigned long flags;
171 +
172 + spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
173 + if (list_empty(&dpaa2_chan->comp_free)) {
174 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
175 + comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
176 + if (!comp_temp)
177 + goto err;
178 + comp_temp->fd_virt_addr = dma_pool_alloc(dpaa2_chan->fd_pool,
179 + GFP_NOWAIT, &comp_temp->fd_bus_addr);
180 + if (!comp_temp->fd_virt_addr)
181 + goto err;
182 +
183 + comp_temp->fl_virt_addr =
184 + (void *)((struct dpaa2_fd *)
185 + comp_temp->fd_virt_addr + 1);
186 + comp_temp->fl_bus_addr = comp_temp->fd_bus_addr +
187 + sizeof(struct dpaa2_fd);
188 + comp_temp->desc_virt_addr =
189 + (void *)((struct dpaa2_frame_list *)
190 + comp_temp->fl_virt_addr + 3);
191 + comp_temp->desc_bus_addr = comp_temp->fl_bus_addr +
192 + sizeof(struct dpaa2_frame_list) * 3;
193 +
194 + comp_temp->qchan = dpaa2_chan;
195 + comp_temp->sg_blk_num = 0;
196 + INIT_LIST_HEAD(&comp_temp->sg_src_head);
197 + INIT_LIST_HEAD(&comp_temp->sg_dst_head);
198 + return comp_temp;
199 + }
200 + comp_temp = list_first_entry(&dpaa2_chan->comp_free,
201 + struct dpaa2_qdma_comp, list);
202 + list_del(&comp_temp->list);
203 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
204 +
205 + comp_temp->qchan = dpaa2_chan;
206 +err:
207 + return comp_temp;
208 +}
209 +
210 +static void dpaa2_qdma_populate_fd(uint32_t format,
211 + struct dpaa2_qdma_comp *dpaa2_comp)
212 +{
213 + struct dpaa2_fd *fd;
214 +
215 + fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr;
216 + memset(fd, 0, sizeof(struct dpaa2_fd));
217 +
218 + /* fd populated */
219 + fd->simple.addr = dpaa2_comp->fl_bus_addr;
220 + /* Bypass memory translation, Frame list format, short length disable */
221 + /* we need to disable BMT if fsl-mc use iova addr */
222 + if (smmu_disable)
223 + fd->simple.bpid = QMAN_FD_BMT_ENABLE;
224 + fd->simple.format_offset = QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE;
225 +
226 + fd->simple.frc = format | QDMA_SER_CTX;
227 +}
228 +
229 +/* first frame list for descriptor buffer */
230 +static void dpaa2_qdma_populate_first_framel(
231 + struct dpaa2_frame_list *f_list,
232 + struct dpaa2_qdma_comp *dpaa2_comp)
233 +{
234 + struct dpaa2_qdma_sd_d *sdd;
235 +
236 + sdd = (struct dpaa2_qdma_sd_d *)dpaa2_comp->desc_virt_addr;
237 + memset(sdd, 0, 2 * (sizeof(*sdd)));
238 + /* source and destination descriptor */
239 + sdd->cmd = QDMA_SD_CMD_RDTTYPE_COHERENT; /* source descriptor CMD */
240 + sdd++;
241 + sdd->cmd = QDMA_DD_CMD_WRTTYPE_COHERENT; /* dest descriptor CMD */
242 +
243 + memset(f_list, 0, sizeof(struct dpaa2_frame_list));
244 + /* first frame list to source descriptor */
245 + f_list->addr_lo = dpaa2_comp->desc_bus_addr;
246 + f_list->addr_hi = (dpaa2_comp->desc_bus_addr >> 32);
247 + f_list->data_len.data_len_sl0 = 0x20; /* source/destination desc len */
248 + f_list->fmt = QDMA_FL_FMT_SBF; /* single buffer frame */
249 + if (smmu_disable)
250 + f_list->bmt = QDMA_FL_BMT_ENABLE; /* bypass memory translation */
251 + f_list->sl = QDMA_FL_SL_LONG; /* long length */
252 + f_list->f = 0; /* not the last frame list */
253 +}
254 +
255 +/* source and destination frame list */
256 +static void dpaa2_qdma_populate_frames(struct dpaa2_frame_list *f_list,
257 + dma_addr_t dst, dma_addr_t src, size_t len, uint8_t fmt)
258 +{
259 + /* source frame list to source buffer */
260 + memset(f_list, 0, sizeof(struct dpaa2_frame_list));
261 + f_list->addr_lo = src;
262 + f_list->addr_hi = (src >> 32);
263 + f_list->data_len.data_len_sl0 = len;
264 + f_list->fmt = fmt; /* single buffer frame or scatter gather frame */
265 + if (smmu_disable)
266 + f_list->bmt = QDMA_FL_BMT_ENABLE; /* bypass memory translation */
267 + f_list->sl = QDMA_FL_SL_LONG; /* long length */
268 + f_list->f = 0; /* not the last frame list */
269 +
270 + f_list++;
271 + /* destination frame list to destination buffer */
272 + memset(f_list, 0, sizeof(struct dpaa2_frame_list));
273 + f_list->addr_lo = dst;
274 + f_list->addr_hi = (dst >> 32);
275 + f_list->data_len.data_len_sl0 = len;
276 + f_list->fmt = fmt; /* single buffer frame or scatter gather frame */
277 + if (smmu_disable)
278 + f_list->bmt = QDMA_FL_BMT_ENABLE; /* bypass memory translation */
279 + f_list->sl = QDMA_FL_SL_LONG; /* long length */
280 + f_list->f = QDMA_FL_F; /* Final bit: 1, for last frame list */
281 +}
282 +
283 +static struct dma_async_tx_descriptor *dpaa2_qdma_prep_memcpy(
284 + struct dma_chan *chan, dma_addr_t dst,
285 + dma_addr_t src, size_t len, unsigned long flags)
286 +{
287 + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
288 + struct dpaa2_qdma_comp *dpaa2_comp;
289 + struct dpaa2_frame_list *f_list;
290 + uint32_t format;
291 +
292 + dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan);
293 +
294 +#ifdef LONG_FORMAT
295 + format = QDMA_FD_LONG_FORMAT;
296 +#else
297 + format = QDMA_FD_SHORT_FORMAT;
298 +#endif
299 + /* populate Frame descriptor */
300 + dpaa2_qdma_populate_fd(format, dpaa2_comp);
301 +
302 + f_list = (struct dpaa2_frame_list *)dpaa2_comp->fl_virt_addr;
303 +
304 +#ifdef LONG_FORMAT
305 + /* first frame list for descriptor buffer (logn format) */
306 + dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp);
307 +
308 + f_list++;
309 +#endif
310 +
311 + dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF);
312 +
313 + return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags);
314 +}
315 +
316 +static struct qdma_sg_blk *dpaa2_qdma_get_sg_blk(
317 + struct dpaa2_qdma_comp *dpaa2_comp,
318 + struct dpaa2_qdma_chan *dpaa2_chan)
319 +{
320 + struct qdma_sg_blk *sg_blk = NULL;
321 + dma_addr_t phy_sgb;
322 + unsigned long flags;
323 +
324 + spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
325 + if (list_empty(&dpaa2_chan->sgb_free)) {
326 + sg_blk = (struct qdma_sg_blk *)dma_pool_alloc(
327 + dpaa2_chan->sg_blk_pool,
328 + GFP_NOWAIT, &phy_sgb);
329 + if (!sg_blk) {
330 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
331 + return sg_blk;
332 + }
333 + sg_blk->blk_virt_addr = (void *)(sg_blk + 1);
334 + sg_blk->blk_bus_addr = phy_sgb + sizeof(*sg_blk);
335 + } else {
336 + sg_blk = list_first_entry(&dpaa2_chan->sgb_free,
337 + struct qdma_sg_blk, list);
338 + list_del(&sg_blk->list);
339 + }
340 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
341 +
342 + return sg_blk;
343 +}
344 +
345 +static uint32_t dpaa2_qdma_populate_sg(struct device *dev,
346 + struct dpaa2_qdma_chan *dpaa2_chan,
347 + struct dpaa2_qdma_comp *dpaa2_comp,
348 + struct scatterlist *dst_sg, u32 dst_nents,
349 + struct scatterlist *src_sg, u32 src_nents)
350 +{
351 + struct dpaa2_qdma_sg *src_sge;
352 + struct dpaa2_qdma_sg *dst_sge;
353 + struct qdma_sg_blk *sg_blk;
354 + struct qdma_sg_blk *sg_blk_dst;
355 + dma_addr_t src;
356 + dma_addr_t dst;
357 + uint32_t num;
358 + uint32_t blocks;
359 + uint32_t len = 0;
360 + uint32_t total_len = 0;
361 + int i, j = 0;
362 +
363 + num = min(dst_nents, src_nents);
364 + blocks = num / (NUM_SG_PER_BLK - 1);
365 + if (num % (NUM_SG_PER_BLK - 1))
366 + blocks += 1;
367 + if (dpaa2_comp->sg_blk_num < blocks) {
368 + len = blocks - dpaa2_comp->sg_blk_num;
369 + for (i = 0; i < len; i++) {
370 + /* source sg blocks */
371 + sg_blk = dpaa2_qdma_get_sg_blk(dpaa2_comp, dpaa2_chan);
372 + if (!sg_blk)
373 + return 0;
374 + list_add_tail(&sg_blk->list, &dpaa2_comp->sg_src_head);
375 + /* destination sg blocks */
376 + sg_blk = dpaa2_qdma_get_sg_blk(dpaa2_comp, dpaa2_chan);
377 + if (!sg_blk)
378 + return 0;
379 + list_add_tail(&sg_blk->list, &dpaa2_comp->sg_dst_head);
380 + }
381 + } else {
382 + len = dpaa2_comp->sg_blk_num - blocks;
383 + for (i = 0; i < len; i++) {
384 + spin_lock(&dpaa2_chan->queue_lock);
385 + /* handle source sg blocks */
386 + sg_blk = list_first_entry(&dpaa2_comp->sg_src_head,
387 + struct qdma_sg_blk, list);
388 + list_del(&sg_blk->list);
389 + list_add_tail(&sg_blk->list, &dpaa2_chan->sgb_free);
390 + /* handle destination sg blocks */
391 + sg_blk = list_first_entry(&dpaa2_comp->sg_dst_head,
392 + struct qdma_sg_blk, list);
393 + list_del(&sg_blk->list);
394 + list_add_tail(&sg_blk->list, &dpaa2_chan->sgb_free);
395 + spin_unlock(&dpaa2_chan->queue_lock);
396 + }
397 + }
398 + dpaa2_comp->sg_blk_num = blocks;
399 +
400 + /* get the first source sg phy address */
401 + sg_blk = list_first_entry(&dpaa2_comp->sg_src_head,
402 + struct qdma_sg_blk, list);
403 + dpaa2_comp->sge_src_bus_addr = sg_blk->blk_bus_addr;
404 + /* get the first destinaiton sg phy address */
405 + sg_blk_dst = list_first_entry(&dpaa2_comp->sg_dst_head,
406 + struct qdma_sg_blk, list);
407 + dpaa2_comp->sge_dst_bus_addr = sg_blk_dst->blk_bus_addr;
408 +
409 + for (i = 0; i < blocks; i++) {
410 + src_sge = (struct dpaa2_qdma_sg *)sg_blk->blk_virt_addr;
411 + dst_sge = (struct dpaa2_qdma_sg *)sg_blk_dst->blk_virt_addr;
412 +
413 + for (j = 0; j < (NUM_SG_PER_BLK - 1); j++) {
414 + len = min(sg_dma_len(dst_sg), sg_dma_len(src_sg));
415 + if (0 == len)
416 + goto fetch;
417 + total_len += len;
418 + src = sg_dma_address(src_sg);
419 + dst = sg_dma_address(dst_sg);
420 +
421 + /* source SG */
422 + src_sge->addr_lo = src;
423 + src_sge->addr_hi = (src >> 32);
424 + src_sge->data_len.data_len_sl0 = len;
425 + src_sge->ctrl.sl = QDMA_SG_SL_LONG;
426 + src_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
427 + /* destination SG */
428 + dst_sge->addr_lo = dst;
429 + dst_sge->addr_hi = (dst >> 32);
430 + dst_sge->data_len.data_len_sl0 = len;
431 + dst_sge->ctrl.sl = QDMA_SG_SL_LONG;
432 + dst_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
433 +fetch:
434 + num--;
435 + if (0 == num) {
436 + src_sge->ctrl.f = QDMA_SG_F;
437 + dst_sge->ctrl.f = QDMA_SG_F;
438 + goto end;
439 + }
440 + dst_sg = sg_next(dst_sg);
441 + src_sg = sg_next(src_sg);
442 + src_sge++;
443 + dst_sge++;
444 + if (j == (NUM_SG_PER_BLK - 2)) {
445 + /* for next blocks, extension */
446 + sg_blk = list_next_entry(sg_blk, list);
447 + sg_blk_dst = list_next_entry(sg_blk_dst, list);
448 + src_sge->addr_lo = sg_blk->blk_bus_addr;
449 + src_sge->addr_hi = sg_blk->blk_bus_addr >> 32;
450 + src_sge->ctrl.sl = QDMA_SG_SL_LONG;
451 + src_sge->ctrl.fmt = QDMA_SG_FMT_SGTE;
452 + dst_sge->addr_lo = sg_blk_dst->blk_bus_addr;
453 + dst_sge->addr_hi =
454 + sg_blk_dst->blk_bus_addr >> 32;
455 + dst_sge->ctrl.sl = QDMA_SG_SL_LONG;
456 + dst_sge->ctrl.fmt = QDMA_SG_FMT_SGTE;
457 + }
458 + }
459 + }
460 +
461 +end:
462 + return total_len;
463 +}
464 +
465 +static struct dma_async_tx_descriptor *dpaa2_qdma_prep_sg(
466 + struct dma_chan *chan,
467 + struct scatterlist *dst_sg, u32 dst_nents,
468 + struct scatterlist *src_sg, u32 src_nents,
469 + unsigned long flags)
470 +{
471 + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
472 + struct dpaa2_qdma_comp *dpaa2_comp;
473 + struct dpaa2_frame_list *f_list;
474 + struct device *dev = dpaa2_chan->qdma->priv->dev;
475 + uint32_t total_len = 0;
476 +
477 + /* basic sanity checks */
478 + if (dst_nents == 0 || src_nents == 0)
479 + return NULL;
480 +
481 + if (dst_sg == NULL || src_sg == NULL)
482 + return NULL;
483 +
484 + /* get the descriptors required */
485 + dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan);
486 +
487 + /* populate Frame descriptor */
488 + dpaa2_qdma_populate_fd(QDMA_FD_LONG_FORMAT, dpaa2_comp);
489 +
490 + /* prepare Scatter gather entry for source and destination */
491 + total_len = dpaa2_qdma_populate_sg(dev, dpaa2_chan,
492 + dpaa2_comp, dst_sg, dst_nents, src_sg, src_nents);
493 +
494 + f_list = (struct dpaa2_frame_list *)dpaa2_comp->fl_virt_addr;
495 + /* first frame list for descriptor buffer */
496 + dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp);
497 + f_list++;
498 + /* prepare Scatter gather entry for source and destination */
499 + /* populate source and destination frame list table */
500 + dpaa2_qdma_populate_frames(f_list, dpaa2_comp->sge_dst_bus_addr,
501 + dpaa2_comp->sge_src_bus_addr,
502 + total_len, QDMA_FL_FMT_SGE);
503 +
504 + return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags);
505 +}
506 +
507 +static enum dma_status dpaa2_qdma_tx_status(struct dma_chan *chan,
508 + dma_cookie_t cookie, struct dma_tx_state *txstate)
509 +{
510 + return dma_cookie_status(chan, cookie, txstate);
511 +}
512 +
513 +static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc)
514 +{
515 +}
516 +
517 +static void dpaa2_qdma_issue_pending(struct dma_chan *chan)
518 +{
519 + struct dpaa2_qdma_comp *dpaa2_comp;
520 + struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan);
521 + struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma;
522 + struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv;
523 + struct virt_dma_desc *vdesc;
524 + struct dpaa2_fd *fd;
525 + int err;
526 + unsigned long flags;
527 +
528 + spin_lock_irqsave(&dpaa2_chan->queue_lock, flags);
529 + spin_lock(&dpaa2_chan->vchan.lock);
530 + if (vchan_issue_pending(&dpaa2_chan->vchan)) {
531 + vdesc = vchan_next_desc(&dpaa2_chan->vchan);
532 + if (!vdesc)
533 + goto err_enqueue;
534 + dpaa2_comp = to_fsl_qdma_comp(vdesc);
535 +
536 + fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr;
537 +
538 + list_del(&vdesc->node);
539 + list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used);
540 +
541 + /* TOBO: priority hard-coded to zero */
542 + err = dpaa2_io_service_enqueue_fq(NULL,
543 + priv->tx_queue_attr[0].fqid, fd);
544 + if (err) {
545 + list_del(&dpaa2_comp->list);
546 + list_add_tail(&dpaa2_comp->list,
547 + &dpaa2_chan->comp_free);
548 + }
549 +
550 + }
551 +err_enqueue:
552 + spin_unlock(&dpaa2_chan->vchan.lock);
553 + spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags);
554 +}
555 +
556 +static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev)
557 +{
558 + struct device *dev = &ls_dev->dev;
559 + struct dpaa2_qdma_priv *priv;
560 + struct dpaa2_qdma_priv_per_prio *ppriv;
561 + uint8_t prio_def = DPDMAI_PRIO_NUM;
562 + int err;
563 + int i;
564 +
565 + priv = dev_get_drvdata(dev);
566 +
567 + priv->dev = dev;
568 + priv->dpqdma_id = ls_dev->obj_desc.id;
569 +
570 + /*Get the handle for the DPDMAI this interface is associate with */
571 + err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle);
572 + if (err) {
573 + dev_err(dev, "dpdmai_open() failed\n");
574 + return err;
575 + }
576 + dev_info(dev, "Opened dpdmai object successfully\n");
577 +
578 + err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
579 + &priv->dpdmai_attr);
580 + if (err) {
581 + dev_err(dev, "dpdmai_get_attributes() failed\n");
582 + return err;
583 + }
584 +
585 + if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) {
586 + dev_err(dev, "DPDMAI major version mismatch\n"
587 + "Found %u.%u, supported version is %u.%u\n",
588 + priv->dpdmai_attr.version.major,
589 + priv->dpdmai_attr.version.minor,
590 + DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
591 + }
592 +
593 + if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) {
594 + dev_err(dev, "DPDMAI minor version mismatch\n"
595 + "Found %u.%u, supported version is %u.%u\n",
596 + priv->dpdmai_attr.version.major,
597 + priv->dpdmai_attr.version.minor,
598 + DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR);
599 + }
600 +
601 + priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def);
602 + ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL);
603 + if (!ppriv) {
604 + dev_err(dev, "kzalloc for ppriv failed\n");
605 + return -1;
606 + }
607 + priv->ppriv = ppriv;
608 +
609 + for (i = 0; i < priv->num_pairs; i++) {
610 + err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
611 + i, &priv->rx_queue_attr[i]);
612 + if (err) {
613 + dev_err(dev, "dpdmai_get_rx_queue() failed\n");
614 + return err;
615 + }
616 + ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;
617 +
618 + err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle,
619 + i, &priv->tx_queue_attr[i]);
620 + if (err) {
621 + dev_err(dev, "dpdmai_get_tx_queue() failed\n");
622 + return err;
623 + }
624 + ppriv->req_fqid = priv->tx_queue_attr[i].fqid;
625 + ppriv->prio = i;
626 + ppriv->priv = priv;
627 + ppriv++;
628 + }
629 +
630 + return 0;
631 +}
632 +
633 +static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx)
634 +{
635 + struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx,
636 + struct dpaa2_qdma_priv_per_prio, nctx);
637 + struct dpaa2_qdma_priv *priv = ppriv->priv;
638 + struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp;
639 + struct dpaa2_qdma_chan *qchan;
640 + const struct dpaa2_fd *fd;
641 + const struct dpaa2_fd *fd_eq;
642 + struct dpaa2_dq *dq;
643 + int err;
644 + int is_last = 0;
645 + uint8_t status;
646 + int i;
647 + int found;
648 + uint32_t n_chans = priv->dpaa2_qdma->n_chans;
649 +
650 + do {
651 + err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,
652 + ppriv->store);
653 + } while (err);
654 +
655 + while (!is_last) {
656 + do {
657 + dq = dpaa2_io_store_next(ppriv->store, &is_last);
658 + } while (!is_last && !dq);
659 + if (!dq) {
660 + dev_err(priv->dev, "FQID returned no valid frames!\n");
661 + continue;
662 + }
663 +
664 + /* obtain FD and process the error */
665 + fd = dpaa2_dq_fd(dq);
666 + status = fd->simple.ctrl & 0xff;
667 + if (status)
668 + dev_err(priv->dev, "FD error occurred\n");
669 + found = 0;
670 + for (i = 0; i < n_chans; i++) {
671 + qchan = &priv->dpaa2_qdma->chans[i];
672 + spin_lock(&qchan->queue_lock);
673 + if (list_empty(&qchan->comp_used)) {
674 + spin_unlock(&qchan->queue_lock);
675 + continue;
676 + }
677 + list_for_each_entry_safe(dpaa2_comp, _comp_tmp,
678 + &qchan->comp_used, list) {
679 + fd_eq = (struct dpaa2_fd *)
680 + dpaa2_comp->fd_virt_addr;
681 +
682 + if (fd_eq->simple.addr ==
683 + fd->simple.addr) {
684 +
685 + list_del(&dpaa2_comp->list);
686 + list_add_tail(&dpaa2_comp->list,
687 + &qchan->comp_free);
688 +
689 + spin_lock(&qchan->vchan.lock);
690 + vchan_cookie_complete(
691 + &dpaa2_comp->vdesc);
692 + spin_unlock(&qchan->vchan.lock);
693 + found = 1;
694 + break;
695 + }
696 + }
697 + spin_unlock(&qchan->queue_lock);
698 + if (found)
699 + break;
700 + }
701 + }
702 +
703 + dpaa2_io_service_rearm(NULL, ctx);
704 +}
705 +
706 +static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv)
707 +{
708 + int err, i, num;
709 + struct device *dev = priv->dev;
710 + struct dpaa2_qdma_priv_per_prio *ppriv;
711 +
712 + num = priv->num_pairs;
713 + ppriv = priv->ppriv;
714 + for (i = 0; i < num; i++) {
715 + ppriv->nctx.is_cdan = 0;
716 + ppriv->nctx.desired_cpu = 1;
717 + ppriv->nctx.id = ppriv->rsp_fqid;
718 + ppriv->nctx.cb = dpaa2_qdma_fqdan_cb;
719 + err = dpaa2_io_service_register(NULL, &ppriv->nctx);
720 + if (err) {
721 + dev_err(dev, "Notification register failed\n");
722 + goto err_service;
723 + }
724 +
725 + ppriv->store = dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE,
726 + dev);
727 + if (!ppriv->store) {
728 + dev_err(dev, "dpaa2_io_store_create() failed\n");
729 + goto err_store;
730 + }
731 +
732 + ppriv++;
733 + }
734 + return 0;
735 +
736 +err_store:
737 + dpaa2_io_service_deregister(NULL, &ppriv->nctx);
738 +err_service:
739 + ppriv--;
740 + while (ppriv >= priv->ppriv) {
741 + dpaa2_io_service_deregister(NULL, &ppriv->nctx);
742 + dpaa2_io_store_destroy(ppriv->store);
743 + ppriv--;
744 + }
745 + return -1;
746 +}
747 +
748 +static void __cold dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv)
749 +{
750 + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
751 + int i;
752 +
753 + for (i = 0; i < priv->num_pairs; i++) {
754 + dpaa2_io_store_destroy(ppriv->store);
755 + ppriv++;
756 + }
757 +}
758 +
759 +static void __cold dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv)
760 +{
761 + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
762 + int i;
763 +
764 + for (i = 0; i < priv->num_pairs; i++) {
765 + dpaa2_io_service_deregister(NULL, &ppriv->nctx);
766 + ppriv++;
767 + }
768 +}
769 +
770 +static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv)
771 +{
772 + int err;
773 + struct dpdmai_rx_queue_cfg rx_queue_cfg;
774 + struct device *dev = priv->dev;
775 + struct dpaa2_qdma_priv_per_prio *ppriv;
776 + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
777 + int i, num;
778 +
779 + num = priv->num_pairs;
780 + ppriv = priv->ppriv;
781 + for (i = 0; i < num; i++) {
782 + rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX |
783 + DPDMAI_QUEUE_OPT_DEST;
784 + rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
785 + rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO;
786 + rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
787 + rx_queue_cfg.dest_cfg.priority = ppriv->prio;
788 + err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle,
789 + rx_queue_cfg.dest_cfg.priority, &rx_queue_cfg);
790 + if (err) {
791 + dev_err(dev, "dpdmai_set_rx_queue() failed\n");
792 + return err;
793 + }
794 +
795 + ppriv++;
796 + }
797 +
798 + return 0;
799 +}
800 +
801 +static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv)
802 +{
803 + int err = 0;
804 + struct device *dev = priv->dev;
805 + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
806 + struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv;
807 + int i;
808 +
809 + for (i = 0; i < priv->num_pairs; i++) {
810 + ppriv->nctx.qman64 = 0;
811 + ppriv->nctx.dpio_id = 0;
812 + ppriv++;
813 + }
814 +
815 + err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle);
816 + if (err)
817 + dev_err(dev, "dpdmai_reset() failed\n");
818 +
819 + return err;
820 +}
821 +
822 +static void __cold dpaa2_dpdmai_free_pool(struct dpaa2_qdma_chan *qchan,
823 + struct list_head *head)
824 +{
825 + struct qdma_sg_blk *sgb_tmp, *_sgb_tmp;
826 + /* free the QDMA SG pool block */
827 + list_for_each_entry_safe(sgb_tmp, _sgb_tmp, head, list) {
828 + sgb_tmp->blk_virt_addr = (void *)((struct qdma_sg_blk *)
829 + sgb_tmp->blk_virt_addr - 1);
830 + sgb_tmp->blk_bus_addr = sgb_tmp->blk_bus_addr
831 + - sizeof(*sgb_tmp);
832 + dma_pool_free(qchan->sg_blk_pool, sgb_tmp->blk_virt_addr,
833 + sgb_tmp->blk_bus_addr);
834 + }
835 +
836 +}
837 +
838 +static void __cold dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan,
839 + struct list_head *head)
840 +{
841 + struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp;
842 + /* free the QDMA comp resource */
843 + list_for_each_entry_safe(comp_tmp, _comp_tmp,
844 + head, list) {
845 + dma_pool_free(qchan->fd_pool,
846 + comp_tmp->fd_virt_addr,
847 + comp_tmp->fd_bus_addr);
848 + /* free the SG source block on comp */
849 + dpaa2_dpdmai_free_pool(qchan, &comp_tmp->sg_src_head);
850 + /* free the SG destination block on comp */
851 + dpaa2_dpdmai_free_pool(qchan, &comp_tmp->sg_dst_head);
852 + list_del(&comp_tmp->list);
853 + kfree(comp_tmp);
854 + }
855 +
856 +}
857 +
858 +static void __cold dpaa2_dpdmai_free_channels(
859 + struct dpaa2_qdma_engine *dpaa2_qdma)
860 +{
861 + struct dpaa2_qdma_chan *qchan;
862 + int num, i;
863 +
864 + num = dpaa2_qdma->n_chans;
865 + for (i = 0; i < num; i++) {
866 + qchan = &dpaa2_qdma->chans[i];
867 + dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used);
868 + dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free);
869 + dpaa2_dpdmai_free_pool(qchan, &qchan->sgb_free);
870 + dma_pool_destroy(qchan->fd_pool);
871 + dma_pool_destroy(qchan->sg_blk_pool);
872 + }
873 +}
874 +
875 +static int dpaa2_dpdmai_alloc_channels(struct dpaa2_qdma_engine *dpaa2_qdma)
876 +{
877 + struct dpaa2_qdma_chan *dpaa2_chan;
878 + struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev;
879 + int i;
880 +
881 + INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels);
882 + for (i = 0; i < dpaa2_qdma->n_chans; i++) {
883 + dpaa2_chan = &dpaa2_qdma->chans[i];
884 + dpaa2_chan->qdma = dpaa2_qdma;
885 + dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc;
886 + vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev);
887 +
888 + dpaa2_chan->fd_pool = dma_pool_create("fd_pool",
889 + dev, FD_POOL_SIZE, 32, 0);
890 + if (!dpaa2_chan->fd_pool)
891 + return -1;
892 + dpaa2_chan->sg_blk_pool = dma_pool_create("sg_blk_pool",
893 + dev, SG_POOL_SIZE, 32, 0);
894 + if (!dpaa2_chan->sg_blk_pool)
895 + return -1;
896 +
897 + spin_lock_init(&dpaa2_chan->queue_lock);
898 + INIT_LIST_HEAD(&dpaa2_chan->comp_used);
899 + INIT_LIST_HEAD(&dpaa2_chan->comp_free);
900 + INIT_LIST_HEAD(&dpaa2_chan->sgb_free);
901 + }
902 + return 0;
903 +}
904 +
905 +static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev)
906 +{
907 + struct dpaa2_qdma_priv *priv;
908 + struct device *dev = &dpdmai_dev->dev;
909 + struct dpaa2_qdma_engine *dpaa2_qdma;
910 + int err;
911 +
912 + priv = kzalloc(sizeof(*priv), GFP_KERNEL);
913 + if (!priv)
914 + return -ENOMEM;
915 + dev_set_drvdata(dev, priv);
916 + priv->dpdmai_dev = dpdmai_dev;
917 +
918 + priv->iommu_domain = iommu_get_domain_for_dev(dev);
919 + if (priv->iommu_domain)
920 + smmu_disable = false;
921 +
922 + /* obtain a MC portal */
923 + err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io);
924 + if (err) {
925 + dev_err(dev, "MC portal allocation failed\n");
926 + goto err_mcportal;
927 + }
928 +
929 + /* DPDMAI initialization */
930 + err = dpaa2_qdma_setup(dpdmai_dev);
931 + if (err) {
932 + dev_err(dev, "dpaa2_dpdmai_setup() failed\n");
933 + goto err_dpdmai_setup;
934 + }
935 +
936 + /* DPIO */
937 + err = dpaa2_qdma_dpio_setup(priv);
938 + if (err) {
939 + dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n");
940 + goto err_dpio_setup;
941 + }
942 +
943 + /* DPDMAI binding to DPIO */
944 + err = dpaa2_dpdmai_bind(priv);
945 + if (err) {
946 + dev_err(dev, "dpaa2_dpdmai_bind() failed\n");
947 + goto err_bind;
948 + }
949 +
950 + /* DPDMAI enable */
951 + err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle);
952 + if (err) {
953 + dev_err(dev, "dpdmai_enable() faile\n");
954 + goto err_enable;
955 + }
956 +
957 + dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL);
958 + if (!dpaa2_qdma) {
959 + err = -ENOMEM;
960 + goto err_eng;
961 + }
962 +
963 + priv->dpaa2_qdma = dpaa2_qdma;
964 + dpaa2_qdma->priv = priv;
965 +
966 + dpaa2_qdma->n_chans = NUM_CH;
967 +
968 + err = dpaa2_dpdmai_alloc_channels(dpaa2_qdma);
969 + if (err) {
970 + dev_err(dev, "QDMA alloc channels faile\n");
971 + goto err_reg;
972 + }
973 +
974 + dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask);
975 + dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask);
976 + dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask);
977 + dma_cap_set(DMA_SG, dpaa2_qdma->dma_dev.cap_mask);
978 +
979 + dpaa2_qdma->dma_dev.dev = dev;
980 + dpaa2_qdma->dma_dev.device_alloc_chan_resources
981 + = dpaa2_qdma_alloc_chan_resources;
982 + dpaa2_qdma->dma_dev.device_free_chan_resources
983 + = dpaa2_qdma_free_chan_resources;
984 + dpaa2_qdma->dma_dev.device_tx_status = dpaa2_qdma_tx_status;
985 + dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy;
986 + dpaa2_qdma->dma_dev.device_prep_dma_sg = dpaa2_qdma_prep_sg;
987 + dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending;
988 +
989 + err = dma_async_device_register(&dpaa2_qdma->dma_dev);
990 + if (err) {
991 + dev_err(dev, "Can't register NXP QDMA engine.\n");
992 + goto err_reg;
993 + }
994 +
995 + return 0;
996 +
997 +err_reg:
998 + dpaa2_dpdmai_free_channels(dpaa2_qdma);
999 + kfree(dpaa2_qdma);
1000 +err_eng:
1001 + dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle);
1002 +err_enable:
1003 + dpaa2_dpdmai_dpio_unbind(priv);
1004 +err_bind:
1005 + dpaa2_dpmai_store_free(priv);
1006 + dpaa2_dpdmai_dpio_free(priv);
1007 +err_dpio_setup:
1008 + dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle);
1009 +err_dpdmai_setup:
1010 + fsl_mc_portal_free(priv->mc_io);
1011 +err_mcportal:
1012 + kfree(priv->ppriv);
1013 + kfree(priv);
1014 + dev_set_drvdata(dev, NULL);
1015 + return err;
1016 +}
1017 +
1018 +static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev)
1019 +{
1020 + struct device *dev;
1021 + struct dpaa2_qdma_priv *priv;
1022 + struct dpaa2_qdma_engine *dpaa2_qdma;
1023 +
1024 + dev = &ls_dev->dev;
1025 + priv = dev_get_drvdata(dev);
1026 + dpaa2_qdma = priv->dpaa2_qdma;
1027 +
1028 + dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle);
1029 + dpaa2_dpdmai_dpio_unbind(priv);
1030 + dpaa2_dpmai_store_free(priv);
1031 + dpaa2_dpdmai_dpio_free(priv);
1032 + dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle);
1033 + fsl_mc_portal_free(priv->mc_io);
1034 + dev_set_drvdata(dev, NULL);
1035 + dpaa2_dpdmai_free_channels(dpaa2_qdma);
1036 +
1037 + dma_async_device_unregister(&dpaa2_qdma->dma_dev);
1038 + kfree(priv);
1039 + kfree(dpaa2_qdma);
1040 +
1041 + return 0;
1042 +}
1043 +
1044 +static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = {
1045 + {
1046 + .vendor = FSL_MC_VENDOR_FREESCALE,
1047 + .obj_type = "dpdmai",
1048 + },
1049 + { .vendor = 0x0 }
1050 +};
1051 +
1052 +static struct fsl_mc_driver dpaa2_qdma_driver = {
1053 + .driver = {
1054 + .name = "dpaa2-qdma",
1055 + .owner = THIS_MODULE,
1056 + },
1057 + .probe = dpaa2_qdma_probe,
1058 + .remove = dpaa2_qdma_remove,
1059 + .match_id_table = dpaa2_qdma_id_table
1060 +};
1061 +
1062 +static int __init dpaa2_qdma_driver_init(void)
1063 +{
1064 + return fsl_mc_driver_register(&(dpaa2_qdma_driver));
1065 +}
1066 +late_initcall(dpaa2_qdma_driver_init);
1067 +
1068 +static void __exit fsl_qdma_exit(void)
1069 +{
1070 + fsl_mc_driver_unregister(&(dpaa2_qdma_driver));
1071 +}
1072 +module_exit(fsl_qdma_exit);
1073 +
1074 +MODULE_DESCRIPTION("NXP DPAA2 qDMA driver");
1075 +MODULE_LICENSE("Dual BSD/GPL");
1076 --- /dev/null
1077 +++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.h
1078 @@ -0,0 +1,262 @@
1079 +/* Copyright 2015 NXP Semiconductor Inc.
1080 + *
1081 + * Redistribution and use in source and binary forms, with or without
1082 + * modification, are permitted provided that the following conditions are met:
1083 + * * Redistributions of source code must retain the above copyright
1084 + * notice, this list of conditions and the following disclaimer.
1085 + * * Redistributions in binary form must reproduce the above copyright
1086 + * notice, this list of conditions and the following disclaimer in the
1087 + * documentation and/or other materials provided with the distribution.
1088 + * * Neither the name of NXP Semiconductor nor the
1089 + * names of its contributors may be used to endorse or promote products
1090 + * derived from this software without specific prior written permission.
1091 + *
1092 + *
1093 + * ALTERNATIVELY, this software may be distributed under the terms of the
1094 + * GNU General Public License ("GPL") as published by the Free Software
1095 + * Foundation, either version 2 of that License or (at your option) any
1096 + * later version.
1097 + *
1098 + * THIS SOFTWARE IS PROVIDED BY NXP Semiconductor ``AS IS'' AND ANY
1099 + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
1100 + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
1101 + * DISCLAIMED. IN NO EVENT SHALL NXP Semiconductor BE LIABLE FOR ANY
1102 + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
1103 + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
1104 + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
1105 + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1106 + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
1107 + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1108 + */
1109 +
1110 +#ifndef __DPAA2_QDMA_H
1111 +#define __DPAA2_QDMA_H
1112 +
1113 +#define LONG_FORMAT 1
1114 +
1115 +#define DPAA2_QDMA_STORE_SIZE 16
1116 +#define NUM_CH 8
1117 +#define NUM_SG_PER_BLK 16
1118 +
1119 +#define QDMA_DMR_OFFSET 0x0
1120 +#define QDMA_DQ_EN (0 << 30)
1121 +#define QDMA_DQ_DIS (1 << 30)
1122 +
1123 +#define QDMA_DSR_M_OFFSET 0x10004
1124 +
1125 +struct dpaa2_qdma_sd_d {
1126 + uint32_t rsv:32;
1127 + union {
1128 + struct {
1129 + uint32_t ssd:12; /* souce stride distance */
1130 + uint32_t sss:12; /* souce stride size */
1131 + uint32_t rsv1:8;
1132 + } sdf;
1133 + struct {
1134 + uint32_t dsd:12; /* Destination stride distance */
1135 + uint32_t dss:12; /* Destination stride size */
1136 + uint32_t rsv2:8;
1137 + } ddf;
1138 + } df;
1139 + uint32_t rbpcmd; /* Route-by-port command */
1140 + uint32_t cmd;
1141 +} __attribute__((__packed__));
1142 +/* Source descriptor command read transaction type for RBP=0:
1143 + coherent copy of cacheable memory */
1144 +#define QDMA_SD_CMD_RDTTYPE_COHERENT (0xb << 28)
1145 +/* Destination descriptor command write transaction type for RBP=0:
1146 + coherent copy of cacheable memory */
1147 +#define QDMA_DD_CMD_WRTTYPE_COHERENT (0x6 << 28)
1148 +
1149 +#define QDMA_SG_FMT_SDB 0x0 /* single data buffer */
1150 +#define QDMA_SG_FMT_FDS 0x1 /* frame data section */
1151 +#define QDMA_SG_FMT_SGTE 0x2 /* SGT extension */
1152 +#define QDMA_SG_SL_SHORT 0x1 /* short length */
1153 +#define QDMA_SG_SL_LONG 0x0 /* short length */
1154 +#define QDMA_SG_F 0x1 /* last sg entry */
1155 +struct dpaa2_qdma_sg {
1156 + uint32_t addr_lo; /* address 0:31 */
1157 + uint32_t addr_hi:17; /* address 32:48 */
1158 + uint32_t rsv:15;
1159 + union {
1160 + uint32_t data_len_sl0; /* SL=0, the long format */
1161 + struct {
1162 + uint32_t len:17; /* SL=1, the short format */
1163 + uint32_t reserve:3;
1164 + uint32_t sf:1;
1165 + uint32_t sr:1;
1166 + uint32_t size:10; /* buff size */
1167 + } data_len_sl1;
1168 + } data_len; /* AVAIL_LENGTH */
1169 + struct {
1170 + uint32_t bpid:14;
1171 + uint32_t ivp:1;
1172 + uint32_t mbt:1;
1173 + uint32_t offset:12;
1174 + uint32_t fmt:2;
1175 + uint32_t sl:1;
1176 + uint32_t f:1;
1177 + } ctrl;
1178 +} __attribute__((__packed__));
1179 +
1180 +#define QMAN_FD_FMT_ENABLE (1 << 12) /* frame list table enable */
1181 +#define QMAN_FD_BMT_ENABLE (1 << 15) /* bypass memory translation */
1182 +#define QMAN_FD_BMT_DISABLE (0 << 15) /* bypass memory translation */
1183 +#define QMAN_FD_SL_DISABLE (0 << 14) /* short lengthe disabled */
1184 +#define QMAN_FD_SL_ENABLE (1 << 14) /* short lengthe enabled */
1185 +
1186 +#define QDMA_SB_FRAME (0 << 28) /* single frame */
1187 +#define QDMA_SG_FRAME (2 << 28) /* scatter gather frames */
1188 +#define QDMA_FINAL_BIT_DISABLE (0 << 31) /* final bit disable */
1189 +#define QDMA_FINAL_BIT_ENABLE (1 << 31) /* final bit enable */
1190 +
1191 +#define QDMA_FD_SHORT_FORMAT (1 << 11) /* short format */
1192 +#define QDMA_FD_LONG_FORMAT (0 << 11) /* long format */
1193 +#define QDMA_SER_DISABLE (0 << 8) /* no notification */
1194 +#define QDMA_SER_CTX (1 << 8) /* notification by FQD_CTX[fqid] */
1195 +#define QDMA_SER_DEST (2 << 8) /* notification by destination desc */
1196 +#define QDMA_SER_BOTH (3 << 8) /* soruce and dest notification */
1197 +#define QDMA_FD_SPF_ENALBE (1 << 30) /* source prefetch enable */
1198 +
1199 +#define QMAN_FD_VA_ENABLE (1 << 14) /* Address used is virtual address */
1200 +#define QMAN_FD_VA_DISABLE (0 << 14)/* Address used is a real address */
1201 +#define QMAN_FD_CBMT_ENABLE (1 << 15) /* Flow Context: 49bit physical address */
1202 +#define QMAN_FD_CBMT_DISABLE (0 << 15) /* Flow Context: 64bit virtual address */
1203 +#define QMAN_FD_SC_DISABLE (0 << 27) /* stashing control */
1204 +
1205 +#define QDMA_FL_FMT_SBF 0x0 /* Single buffer frame */
1206 +#define QDMA_FL_FMT_SGE 0x2 /* Scatter gather frame */
1207 +#define QDMA_FL_BMT_ENABLE 0x1 /* enable bypass memory translation */
1208 +#define QDMA_FL_BMT_DISABLE 0x0 /* enable bypass memory translation */
1209 +#define QDMA_FL_SL_LONG 0x0 /* long length */
1210 +#define QDMA_FL_SL_SHORT 0x1 /* short length */
1211 +#define QDMA_FL_F 0x1 /* last frame list bit */
1212 +/*Description of Frame list table structure*/
1213 +struct dpaa2_frame_list {
1214 + uint32_t addr_lo; /* lower 32 bits of address */
1215 + uint32_t addr_hi:17; /* upper 17 bits of address */
1216 + uint32_t resrvd:15;
1217 + union {
1218 + uint32_t data_len_sl0; /* If SL=0, then data length is 32 */
1219 + struct {
1220 + uint32_t data_len:18; /* IF SL=1; length is 18bit */
1221 + uint32_t resrvd:2;
1222 + uint32_t mem:12; /* Valid only when SL=1 */
1223 + } data_len_sl1;
1224 + } data_len;
1225 + /* word 4 */
1226 + uint32_t bpid:14; /* Frame buffer pool ID */
1227 + uint32_t ivp:1; /* Invalid Pool ID. */
1228 + uint32_t bmt:1; /* Bypass Memory Translation */
1229 + uint32_t offset:12; /* Frame offset */
1230 + uint32_t fmt:2; /* Frame Format */
1231 + uint32_t sl:1; /* Short Length */
1232 + uint32_t f:1; /* Final bit */
1233 +
1234 + uint32_t frc; /* Frame Context */
1235 + /* word 6 */
1236 + uint32_t err:8; /* Frame errors */
1237 + uint32_t resrvd0:8;
1238 + uint32_t asal:4; /* accelerator-specific annotation length */
1239 + uint32_t resrvd1:1;
1240 + uint32_t ptv2:1;
1241 + uint32_t ptv1:1;
1242 + uint32_t pta:1; /* pass-through annotation */
1243 + uint32_t resrvd2:8;
1244 +
1245 + uint32_t flc_lo; /* lower 32 bits fo flow context */
1246 + uint32_t flc_hi; /* higher 32 bits fo flow context */
1247 +} __attribute__((__packed__));
1248 +
1249 +struct dpaa2_qdma_chan {
1250 + struct virt_dma_chan vchan;
1251 + struct virt_dma_desc vdesc;
1252 + enum dma_status status;
1253 + struct dpaa2_qdma_engine *qdma;
1254 +
1255 + struct mutex dpaa2_queue_mutex;
1256 + spinlock_t queue_lock;
1257 + struct dma_pool *fd_pool;
1258 + struct dma_pool *sg_blk_pool;
1259 +
1260 + struct list_head comp_used;
1261 + struct list_head comp_free;
1262 +
1263 + struct list_head sgb_free;
1264 +};
1265 +
1266 +struct qdma_sg_blk {
1267 + dma_addr_t blk_bus_addr;
1268 + void *blk_virt_addr;
1269 + struct list_head list;
1270 +};
1271 +
1272 +struct dpaa2_qdma_comp {
1273 + dma_addr_t fd_bus_addr;
1274 + dma_addr_t fl_bus_addr;
1275 + dma_addr_t desc_bus_addr;
1276 + dma_addr_t sge_src_bus_addr;
1277 + dma_addr_t sge_dst_bus_addr;
1278 + void *fd_virt_addr;
1279 + void *fl_virt_addr;
1280 + void *desc_virt_addr;
1281 + void *sg_src_virt_addr;
1282 + void *sg_dst_virt_addr;
1283 + struct qdma_sg_blk *sg_blk;
1284 + uint32_t sg_blk_num;
1285 + struct list_head sg_src_head;
1286 + struct list_head sg_dst_head;
1287 + struct dpaa2_qdma_chan *qchan;
1288 + struct virt_dma_desc vdesc;
1289 + struct list_head list;
1290 +};
1291 +
1292 +struct dpaa2_qdma_engine {
1293 + struct dma_device dma_dev;
1294 + u32 n_chans;
1295 + struct dpaa2_qdma_chan chans[NUM_CH];
1296 +
1297 + struct dpaa2_qdma_priv *priv;
1298 +};
1299 +
1300 +/*
1301 + * dpaa2_qdma_priv - driver private data
1302 + */
1303 +struct dpaa2_qdma_priv {
1304 + int dpqdma_id;
1305 +
1306 + struct iommu_domain *iommu_domain;
1307 + struct dpdmai_attr dpdmai_attr;
1308 + struct device *dev;
1309 + struct fsl_mc_io *mc_io;
1310 + struct fsl_mc_device *dpdmai_dev;
1311 +
1312 + struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM];
1313 + struct dpdmai_tx_queue_attr tx_queue_attr[DPDMAI_PRIO_NUM];
1314 +
1315 + uint8_t num_pairs;
1316 +
1317 + struct dpaa2_qdma_engine *dpaa2_qdma;
1318 + struct dpaa2_qdma_priv_per_prio *ppriv;
1319 +};
1320 +
1321 +struct dpaa2_qdma_priv_per_prio {
1322 + int req_fqid;
1323 + int rsp_fqid;
1324 + int prio;
1325 +
1326 + struct dpaa2_io_store *store;
1327 + struct dpaa2_io_notification_ctx nctx;
1328 +
1329 + struct dpaa2_qdma_priv *priv;
1330 +};
1331 +
1332 +/* FD pool size: one FD + 3 Frame list + 2 source/destination descriptor */
1333 +#define FD_POOL_SIZE (sizeof(struct dpaa2_fd) + \
1334 + sizeof(struct dpaa2_frame_list) * 3 + \
1335 + sizeof(struct dpaa2_qdma_sd_d) * 2)
1336 +
1337 +/* qdma_sg_blk + 16 SGs */
1338 +#define SG_POOL_SIZE (sizeof(struct qdma_sg_blk) +\
1339 + sizeof(struct dpaa2_qdma_sg) * NUM_SG_PER_BLK)
1340 +#endif /* __DPAA2_QDMA_H */
1341 --- /dev/null
1342 +++ b/drivers/dma/dpaa2-qdma/dpdmai.c
1343 @@ -0,0 +1,454 @@
1344 +/* Copyright 2013-2015 Freescale Semiconductor Inc.
1345 + *
1346 + * Redistribution and use in source and binary forms, with or without
1347 + * modification, are permitted provided that the following conditions are met:
1348 + * * Redistributions of source code must retain the above copyright
1349 + * notice, this list of conditions and the following disclaimer.
1350 + * * Redistributions in binary form must reproduce the above copyright
1351 + * notice, this list of conditions and the following disclaimer in the
1352 + * documentation and/or other materials provided with the distribution.
1353 + * * Neither the name of the above-listed copyright holders nor the
1354 + * names of any contributors may be used to endorse or promote products
1355 + * derived from this software without specific prior written permission.
1356 + *
1357 + *
1358 + * ALTERNATIVELY, this software may be distributed under the terms of the
1359 + * GNU General Public License ("GPL") as published by the Free Software
1360 + * Foundation, either version 2 of that License or (at your option) any
1361 + * later version.
1362 + *
1363 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1364 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1365 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1366 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
1367 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
1368 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
1369 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
1370 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
1371 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
1372 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
1373 + * POSSIBILITY OF SUCH DAMAGE.
1374 + */
1375 +#include <linux/types.h>
1376 +#include <linux/io.h>
1377 +#include "fsl_dpdmai.h"
1378 +#include "fsl_dpdmai_cmd.h"
1379 +#include "../../../drivers/staging/fsl-mc/include/mc-sys.h"
1380 +#include "../../../drivers/staging/fsl-mc/include/mc-cmd.h"
1381 +
1382 +int dpdmai_open(struct fsl_mc_io *mc_io,
1383 + uint32_t cmd_flags,
1384 + int dpdmai_id,
1385 + uint16_t *token)
1386 +{
1387 + struct mc_command cmd = { 0 };
1388 + int err;
1389 +
1390 + /* prepare command */
1391 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN,
1392 + cmd_flags,
1393 + 0);
1394 + DPDMAI_CMD_OPEN(cmd, dpdmai_id);
1395 +
1396 + /* send command to mc*/
1397 + err = mc_send_command(mc_io, &cmd);
1398 + if (err)
1399 + return err;
1400 +
1401 + /* retrieve response parameters */
1402 + *token = MC_CMD_HDR_READ_TOKEN(cmd.header);
1403 +
1404 + return 0;
1405 +}
1406 +
1407 +int dpdmai_close(struct fsl_mc_io *mc_io,
1408 + uint32_t cmd_flags,
1409 + uint16_t token)
1410 +{
1411 + struct mc_command cmd = { 0 };
1412 +
1413 + /* prepare command */
1414 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLOSE,
1415 + cmd_flags, token);
1416 +
1417 + /* send command to mc*/
1418 + return mc_send_command(mc_io, &cmd);
1419 +}
1420 +
1421 +int dpdmai_create(struct fsl_mc_io *mc_io,
1422 + uint32_t cmd_flags,
1423 + const struct dpdmai_cfg *cfg,
1424 + uint16_t *token)
1425 +{
1426 + struct mc_command cmd = { 0 };
1427 + int err;
1428 +
1429 + /* prepare command */
1430 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE,
1431 + cmd_flags,
1432 + 0);
1433 + DPDMAI_CMD_CREATE(cmd, cfg);
1434 +
1435 + /* send command to mc*/
1436 + err = mc_send_command(mc_io, &cmd);
1437 + if (err)
1438 + return err;
1439 +
1440 + /* retrieve response parameters */
1441 + *token = MC_CMD_HDR_READ_TOKEN(cmd.header);
1442 +
1443 + return 0;
1444 +}
1445 +
1446 +int dpdmai_destroy(struct fsl_mc_io *mc_io,
1447 + uint32_t cmd_flags,
1448 + uint16_t token)
1449 +{
1450 + struct mc_command cmd = { 0 };
1451 +
1452 + /* prepare command */
1453 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY,
1454 + cmd_flags,
1455 + token);
1456 +
1457 + /* send command to mc*/
1458 + return mc_send_command(mc_io, &cmd);
1459 +}
1460 +
1461 +int dpdmai_enable(struct fsl_mc_io *mc_io,
1462 + uint32_t cmd_flags,
1463 + uint16_t token)
1464 +{
1465 + struct mc_command cmd = { 0 };
1466 +
1467 + /* prepare command */
1468 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_ENABLE,
1469 + cmd_flags,
1470 + token);
1471 +
1472 + /* send command to mc*/
1473 + return mc_send_command(mc_io, &cmd);
1474 +}
1475 +
1476 +int dpdmai_disable(struct fsl_mc_io *mc_io,
1477 + uint32_t cmd_flags,
1478 + uint16_t token)
1479 +{
1480 + struct mc_command cmd = { 0 };
1481 +
1482 + /* prepare command */
1483 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DISABLE,
1484 + cmd_flags,
1485 + token);
1486 +
1487 + /* send command to mc*/
1488 + return mc_send_command(mc_io, &cmd);
1489 +}
1490 +
1491 +int dpdmai_is_enabled(struct fsl_mc_io *mc_io,
1492 + uint32_t cmd_flags,
1493 + uint16_t token,
1494 + int *en)
1495 +{
1496 + struct mc_command cmd = { 0 };
1497 + int err;
1498 + /* prepare command */
1499 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_IS_ENABLED,
1500 + cmd_flags,
1501 + token);
1502 +
1503 + /* send command to mc*/
1504 + err = mc_send_command(mc_io, &cmd);
1505 + if (err)
1506 + return err;
1507 +
1508 + /* retrieve response parameters */
1509 + DPDMAI_RSP_IS_ENABLED(cmd, *en);
1510 +
1511 + return 0;
1512 +}
1513 +
1514 +int dpdmai_reset(struct fsl_mc_io *mc_io,
1515 + uint32_t cmd_flags,
1516 + uint16_t token)
1517 +{
1518 + struct mc_command cmd = { 0 };
1519 +
1520 + /* prepare command */
1521 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_RESET,
1522 + cmd_flags,
1523 + token);
1524 +
1525 + /* send command to mc*/
1526 + return mc_send_command(mc_io, &cmd);
1527 +}
1528 +
1529 +int dpdmai_get_irq(struct fsl_mc_io *mc_io,
1530 + uint32_t cmd_flags,
1531 + uint16_t token,
1532 + uint8_t irq_index,
1533 + int *type,
1534 + struct dpdmai_irq_cfg *irq_cfg)
1535 +{
1536 + struct mc_command cmd = { 0 };
1537 + int err;
1538 +
1539 + /* prepare command */
1540 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ,
1541 + cmd_flags,
1542 + token);
1543 + DPDMAI_CMD_GET_IRQ(cmd, irq_index);
1544 +
1545 + /* send command to mc*/
1546 + err = mc_send_command(mc_io, &cmd);
1547 + if (err)
1548 + return err;
1549 +
1550 + /* retrieve response parameters */
1551 + DPDMAI_RSP_GET_IRQ(cmd, *type, irq_cfg);
1552 +
1553 + return 0;
1554 +}
1555 +
1556 +int dpdmai_set_irq(struct fsl_mc_io *mc_io,
1557 + uint32_t cmd_flags,
1558 + uint16_t token,
1559 + uint8_t irq_index,
1560 + struct dpdmai_irq_cfg *irq_cfg)
1561 +{
1562 + struct mc_command cmd = { 0 };
1563 +
1564 + /* prepare command */
1565 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ,
1566 + cmd_flags,
1567 + token);
1568 + DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg);
1569 +
1570 + /* send command to mc*/
1571 + return mc_send_command(mc_io, &cmd);
1572 +}
1573 +
1574 +int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io,
1575 + uint32_t cmd_flags,
1576 + uint16_t token,
1577 + uint8_t irq_index,
1578 + uint8_t *en)
1579 +{
1580 + struct mc_command cmd = { 0 };
1581 + int err;
1582 +
1583 + /* prepare command */
1584 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_ENABLE,
1585 + cmd_flags,
1586 + token);
1587 + DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index);
1588 +
1589 + /* send command to mc*/
1590 + err = mc_send_command(mc_io, &cmd);
1591 + if (err)
1592 + return err;
1593 +
1594 + /* retrieve response parameters */
1595 + DPDMAI_RSP_GET_IRQ_ENABLE(cmd, *en);
1596 +
1597 + return 0;
1598 +}
1599 +
1600 +int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io,
1601 + uint32_t cmd_flags,
1602 + uint16_t token,
1603 + uint8_t irq_index,
1604 + uint8_t en)
1605 +{
1606 + struct mc_command cmd = { 0 };
1607 +
1608 + /* prepare command */
1609 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_ENABLE,
1610 + cmd_flags,
1611 + token);
1612 + DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, en);
1613 +
1614 + /* send command to mc*/
1615 + return mc_send_command(mc_io, &cmd);
1616 +}
1617 +
1618 +int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io,
1619 + uint32_t cmd_flags,
1620 + uint16_t token,
1621 + uint8_t irq_index,
1622 + uint32_t *mask)
1623 +{
1624 + struct mc_command cmd = { 0 };
1625 + int err;
1626 +
1627 + /* prepare command */
1628 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_MASK,
1629 + cmd_flags,
1630 + token);
1631 + DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index);
1632 +
1633 + /* send command to mc*/
1634 + err = mc_send_command(mc_io, &cmd);
1635 + if (err)
1636 + return err;
1637 +
1638 + /* retrieve response parameters */
1639 + DPDMAI_RSP_GET_IRQ_MASK(cmd, *mask);
1640 +
1641 + return 0;
1642 +}
1643 +
1644 +int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io,
1645 + uint32_t cmd_flags,
1646 + uint16_t token,
1647 + uint8_t irq_index,
1648 + uint32_t mask)
1649 +{
1650 + struct mc_command cmd = { 0 };
1651 +
1652 + /* prepare command */
1653 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_MASK,
1654 + cmd_flags,
1655 + token);
1656 + DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask);
1657 +
1658 + /* send command to mc*/
1659 + return mc_send_command(mc_io, &cmd);
1660 +}
1661 +
1662 +int dpdmai_get_irq_status(struct fsl_mc_io *mc_io,
1663 + uint32_t cmd_flags,
1664 + uint16_t token,
1665 + uint8_t irq_index,
1666 + uint32_t *status)
1667 +{
1668 + struct mc_command cmd = { 0 };
1669 + int err;
1670 +
1671 + /* prepare command */
1672 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_STATUS,
1673 + cmd_flags,
1674 + token);
1675 + DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, *status);
1676 +
1677 + /* send command to mc*/
1678 + err = mc_send_command(mc_io, &cmd);
1679 + if (err)
1680 + return err;
1681 +
1682 + /* retrieve response parameters */
1683 + DPDMAI_RSP_GET_IRQ_STATUS(cmd, *status);
1684 +
1685 + return 0;
1686 +}
1687 +
1688 +int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io,
1689 + uint32_t cmd_flags,
1690 + uint16_t token,
1691 + uint8_t irq_index,
1692 + uint32_t status)
1693 +{
1694 + struct mc_command cmd = { 0 };
1695 +
1696 + /* prepare command */
1697 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLEAR_IRQ_STATUS,
1698 + cmd_flags,
1699 + token);
1700 + DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status);
1701 +
1702 + /* send command to mc*/
1703 + return mc_send_command(mc_io, &cmd);
1704 +}
1705 +
1706 +int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
1707 + uint32_t cmd_flags,
1708 + uint16_t token,
1709 + struct dpdmai_attr *attr)
1710 +{
1711 + struct mc_command cmd = { 0 };
1712 + int err;
1713 +
1714 + /* prepare command */
1715 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_ATTR,
1716 + cmd_flags,
1717 + token);
1718 +
1719 + /* send command to mc*/
1720 + err = mc_send_command(mc_io, &cmd);
1721 + if (err)
1722 + return err;
1723 +
1724 + /* retrieve response parameters */
1725 + DPDMAI_RSP_GET_ATTR(cmd, attr);
1726 +
1727 + return 0;
1728 +}
1729 +
1730 +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
1731 + uint32_t cmd_flags,
1732 + uint16_t token,
1733 + uint8_t priority,
1734 + const struct dpdmai_rx_queue_cfg *cfg)
1735 +{
1736 + struct mc_command cmd = { 0 };
1737 +
1738 + /* prepare command */
1739 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_RX_QUEUE,
1740 + cmd_flags,
1741 + token);
1742 + DPDMAI_CMD_SET_RX_QUEUE(cmd, priority, cfg);
1743 +
1744 + /* send command to mc*/
1745 + return mc_send_command(mc_io, &cmd);
1746 +}
1747 +
1748 +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
1749 + uint32_t cmd_flags,
1750 + uint16_t token,
1751 + uint8_t priority, struct dpdmai_rx_queue_attr *attr)
1752 +{
1753 + struct mc_command cmd = { 0 };
1754 + int err;
1755 +
1756 + /* prepare command */
1757 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_RX_QUEUE,
1758 + cmd_flags,
1759 + token);
1760 + DPDMAI_CMD_GET_RX_QUEUE(cmd, priority);
1761 +
1762 + /* send command to mc*/
1763 + err = mc_send_command(mc_io, &cmd);
1764 + if (err)
1765 + return err;
1766 +
1767 + /* retrieve response parameters */
1768 + DPDMAI_RSP_GET_RX_QUEUE(cmd, attr);
1769 +
1770 + return 0;
1771 +}
1772 +
1773 +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
1774 + uint32_t cmd_flags,
1775 + uint16_t token,
1776 + uint8_t priority,
1777 + struct dpdmai_tx_queue_attr *attr)
1778 +{
1779 + struct mc_command cmd = { 0 };
1780 + int err;
1781 +
1782 + /* prepare command */
1783 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_TX_QUEUE,
1784 + cmd_flags,
1785 + token);
1786 + DPDMAI_CMD_GET_TX_QUEUE(cmd, priority);
1787 +
1788 + /* send command to mc*/
1789 + err = mc_send_command(mc_io, &cmd);
1790 + if (err)
1791 + return err;
1792 +
1793 + /* retrieve response parameters */
1794 + DPDMAI_RSP_GET_TX_QUEUE(cmd, attr);
1795 +
1796 + return 0;
1797 +}
1798 --- /dev/null
1799 +++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai.h
1800 @@ -0,0 +1,521 @@
1801 +/* Copyright 2013-2015 Freescale Semiconductor Inc.
1802 + *
1803 + * Redistribution and use in source and binary forms, with or without
1804 + * modification, are permitted provided that the following conditions are met:
1805 + * * Redistributions of source code must retain the above copyright
1806 + * notice, this list of conditions and the following disclaimer.
1807 + * * Redistributions in binary form must reproduce the above copyright
1808 + * notice, this list of conditions and the following disclaimer in the
1809 + * documentation and/or other materials provided with the distribution.
1810 + * * Neither the name of the above-listed copyright holders nor the
1811 + * names of any contributors may be used to endorse or promote products
1812 + * derived from this software without specific prior written permission.
1813 + *
1814 + *
1815 + * ALTERNATIVELY, this software may be distributed under the terms of the
1816 + * GNU General Public License ("GPL") as published by the Free Software
1817 + * Foundation, either version 2 of that License or (at your option) any
1818 + * later version.
1819 + *
1820 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1821 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1822 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1823 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
1824 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
1825 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
1826 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
1827 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
1828 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
1829 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
1830 + * POSSIBILITY OF SUCH DAMAGE.
1831 + */
1832 +#ifndef __FSL_DPDMAI_H
1833 +#define __FSL_DPDMAI_H
1834 +
1835 +struct fsl_mc_io;
1836 +
1837 +/* Data Path DMA Interface API
1838 + * Contains initialization APIs and runtime control APIs for DPDMAI
1839 + */
1840 +
1841 +/* General DPDMAI macros */
1842 +
1843 +/**
1844 + * Maximum number of Tx/Rx priorities per DPDMAI object
1845 + */
1846 +#define DPDMAI_PRIO_NUM 2
1847 +
1848 +/**
1849 + * All queues considered; see dpdmai_set_rx_queue()
1850 + */
1851 +#define DPDMAI_ALL_QUEUES (uint8_t)(-1)
1852 +
1853 +/**
1854 + * dpdmai_open() - Open a control session for the specified object
1855 + * @mc_io: Pointer to MC portal's I/O object
1856 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
1857 + * @dpdmai_id: DPDMAI unique ID
1858 + * @token: Returned token; use in subsequent API calls
1859 + *
1860 + * This function can be used to open a control session for an
1861 + * already created object; an object may have been declared in
1862 + * the DPL or by calling the dpdmai_create() function.
1863 + * This function returns a unique authentication token,
1864 + * associated with the specific object ID and the specific MC
1865 + * portal; this token must be used in all subsequent commands for
1866 + * this specific object.
1867 + *
1868 + * Return: '0' on Success; Error code otherwise.
1869 + */
1870 +int dpdmai_open(struct fsl_mc_io *mc_io,
1871 + uint32_t cmd_flags,
1872 + int dpdmai_id,
1873 + uint16_t *token);
1874 +
1875 +/**
1876 + * dpdmai_close() - Close the control session of the object
1877 + * @mc_io: Pointer to MC portal's I/O object
1878 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
1879 + * @token: Token of DPDMAI object
1880 + *
1881 + * After this function is called, no further operations are
1882 + * allowed on the object without opening a new control session.
1883 + *
1884 + * Return: '0' on Success; Error code otherwise.
1885 + */
1886 +int dpdmai_close(struct fsl_mc_io *mc_io,
1887 + uint32_t cmd_flags,
1888 + uint16_t token);
1889 +
1890 +/**
1891 + * struct dpdmai_cfg - Structure representing DPDMAI configuration
1892 + * @priorities: Priorities for the DMA hardware processing; valid priorities are
1893 + * configured with values 1-8; the entry following last valid entry
1894 + * should be configured with 0
1895 + */
1896 +struct dpdmai_cfg {
1897 + uint8_t priorities[DPDMAI_PRIO_NUM];
1898 +};
1899 +
1900 +/**
1901 + * dpdmai_create() - Create the DPDMAI object
1902 + * @mc_io: Pointer to MC portal's I/O object
1903 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
1904 + * @cfg: Configuration structure
1905 + * @token: Returned token; use in subsequent API calls
1906 + *
1907 + * Create the DPDMAI object, allocate required resources and
1908 + * perform required initialization.
1909 + *
1910 + * The object can be created either by declaring it in the
1911 + * DPL file, or by calling this function.
1912 + *
1913 + * This function returns a unique authentication token,
1914 + * associated with the specific object ID and the specific MC
1915 + * portal; this token must be used in all subsequent calls to
1916 + * this specific object. For objects that are created using the
1917 + * DPL file, call dpdmai_open() function to get an authentication
1918 + * token first.
1919 + *
1920 + * Return: '0' on Success; Error code otherwise.
1921 + */
1922 +int dpdmai_create(struct fsl_mc_io *mc_io,
1923 + uint32_t cmd_flags,
1924 + const struct dpdmai_cfg *cfg,
1925 + uint16_t *token);
1926 +
1927 +/**
1928 + * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources.
1929 + * @mc_io: Pointer to MC portal's I/O object
1930 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
1931 + * @token: Token of DPDMAI object
1932 + *
1933 + * Return: '0' on Success; error code otherwise.
1934 + */
1935 +int dpdmai_destroy(struct fsl_mc_io *mc_io,
1936 + uint32_t cmd_flags,
1937 + uint16_t token);
1938 +
1939 +/**
1940 + * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames.
1941 + * @mc_io: Pointer to MC portal's I/O object
1942 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
1943 + * @token: Token of DPDMAI object
1944 + *
1945 + * Return: '0' on Success; Error code otherwise.
1946 + */
1947 +int dpdmai_enable(struct fsl_mc_io *mc_io,
1948 + uint32_t cmd_flags,
1949 + uint16_t token);
1950 +
1951 +/**
1952 + * dpdmai_disable() - Disable the DPDMAI, stop sending and receiving frames.
1953 + * @mc_io: Pointer to MC portal's I/O object
1954 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
1955 + * @token: Token of DPDMAI object
1956 + *
1957 + * Return: '0' on Success; Error code otherwise.
1958 + */
1959 +int dpdmai_disable(struct fsl_mc_io *mc_io,
1960 + uint32_t cmd_flags,
1961 + uint16_t token);
1962 +
1963 +/**
1964 + * dpdmai_is_enabled() - Check if the DPDMAI is enabled.
1965 + * @mc_io: Pointer to MC portal's I/O object
1966 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
1967 + * @token: Token of DPDMAI object
1968 + * @en: Returns '1' if object is enabled; '0' otherwise
1969 + *
1970 + * Return: '0' on Success; Error code otherwise.
1971 + */
1972 +int dpdmai_is_enabled(struct fsl_mc_io *mc_io,
1973 + uint32_t cmd_flags,
1974 + uint16_t token,
1975 + int *en);
1976 +
1977 +/**
1978 + * dpdmai_reset() - Reset the DPDMAI, returns the object to initial state.
1979 + * @mc_io: Pointer to MC portal's I/O object
1980 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
1981 + * @token: Token of DPDMAI object
1982 + *
1983 + * Return: '0' on Success; Error code otherwise.
1984 + */
1985 +int dpdmai_reset(struct fsl_mc_io *mc_io,
1986 + uint32_t cmd_flags,
1987 + uint16_t token);
1988 +
1989 +/**
1990 + * struct dpdmai_irq_cfg - IRQ configuration
1991 + * @addr: Address that must be written to signal a message-based interrupt
1992 + * @val: Value to write into irq_addr address
1993 + * @irq_num: A user defined number associated with this IRQ
1994 + */
1995 +struct dpdmai_irq_cfg {
1996 + uint64_t addr;
1997 + uint32_t val;
1998 + int irq_num;
1999 +};
2000 +
2001 +/**
2002 + * dpdmai_set_irq() - Set IRQ information for the DPDMAI to trigger an interrupt.
2003 + * @mc_io: Pointer to MC portal's I/O object
2004 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2005 + * @token: Token of DPDMAI object
2006 + * @irq_index: Identifies the interrupt index to configure
2007 + * @irq_cfg: IRQ configuration
2008 + *
2009 + * Return: '0' on Success; Error code otherwise.
2010 + */
2011 +int dpdmai_set_irq(struct fsl_mc_io *mc_io,
2012 + uint32_t cmd_flags,
2013 + uint16_t token,
2014 + uint8_t irq_index,
2015 + struct dpdmai_irq_cfg *irq_cfg);
2016 +
2017 +/**
2018 + * dpdmai_get_irq() - Get IRQ information from the DPDMAI
2019 + *
2020 + * @mc_io: Pointer to MC portal's I/O object
2021 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2022 + * @token: Token of DPDMAI object
2023 + * @irq_index: The interrupt index to configure
2024 + * @type: Interrupt type: 0 represents message interrupt
2025 + * type (both irq_addr and irq_val are valid)
2026 + * @irq_cfg: IRQ attributes
2027 + *
2028 + * Return: '0' on Success; Error code otherwise.
2029 + */
2030 +int dpdmai_get_irq(struct fsl_mc_io *mc_io,
2031 + uint32_t cmd_flags,
2032 + uint16_t token,
2033 + uint8_t irq_index,
2034 + int *type,
2035 + struct dpdmai_irq_cfg *irq_cfg);
2036 +
2037 +/**
2038 + * dpdmai_set_irq_enable() - Set overall interrupt state.
2039 + * @mc_io: Pointer to MC portal's I/O object
2040 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2041 + * @token: Token of DPDMAI object
2042 + * @irq_index: The interrupt index to configure
2043 + * @en: Interrupt state - enable = 1, disable = 0
2044 + *
2045 + * Allows GPP software to control when interrupts are generated.
2046 + * Each interrupt can have up to 32 causes. The enable/disable control's the
2047 + * overall interrupt state. if the interrupt is disabled no causes will cause
2048 + * an interrupt
2049 + *
2050 + * Return: '0' on Success; Error code otherwise.
2051 + */
2052 +int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io,
2053 + uint32_t cmd_flags,
2054 + uint16_t token,
2055 + uint8_t irq_index,
2056 + uint8_t en);
2057 +
2058 +/**
2059 + * dpdmai_get_irq_enable() - Get overall interrupt state
2060 + * @mc_io: Pointer to MC portal's I/O object
2061 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2062 + * @token: Token of DPDMAI object
2063 + * @irq_index: The interrupt index to configure
2064 + * @en: Returned Interrupt state - enable = 1, disable = 0
2065 + *
2066 + * Return: '0' on Success; Error code otherwise.
2067 + */
2068 +int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io,
2069 + uint32_t cmd_flags,
2070 + uint16_t token,
2071 + uint8_t irq_index,
2072 + uint8_t *en);
2073 +
2074 +/**
2075 + * dpdmai_set_irq_mask() - Set interrupt mask.
2076 + * @mc_io: Pointer to MC portal's I/O object
2077 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2078 + * @token: Token of DPDMAI object
2079 + * @irq_index: The interrupt index to configure
2080 + * @mask: event mask to trigger interrupt;
2081 + * each bit:
2082 + * 0 = ignore event
2083 + * 1 = consider event for asserting IRQ
2084 + *
2085 + * Every interrupt can have up to 32 causes and the interrupt model supports
2086 + * masking/unmasking each cause independently
2087 + *
2088 + * Return: '0' on Success; Error code otherwise.
2089 + */
2090 +int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io,
2091 + uint32_t cmd_flags,
2092 + uint16_t token,
2093 + uint8_t irq_index,
2094 + uint32_t mask);
2095 +
2096 +/**
2097 + * dpdmai_get_irq_mask() - Get interrupt mask.
2098 + * @mc_io: Pointer to MC portal's I/O object
2099 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2100 + * @token: Token of DPDMAI object
2101 + * @irq_index: The interrupt index to configure
2102 + * @mask: Returned event mask to trigger interrupt
2103 + *
2104 + * Every interrupt can have up to 32 causes and the interrupt model supports
2105 + * masking/unmasking each cause independently
2106 + *
2107 + * Return: '0' on Success; Error code otherwise.
2108 + */
2109 +int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io,
2110 + uint32_t cmd_flags,
2111 + uint16_t token,
2112 + uint8_t irq_index,
2113 + uint32_t *mask);
2114 +
2115 +/**
2116 + * dpdmai_get_irq_status() - Get the current status of any pending interrupts
2117 + * @mc_io: Pointer to MC portal's I/O object
2118 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2119 + * @token: Token of DPDMAI object
2120 + * @irq_index: The interrupt index to configure
2121 + * @status: Returned interrupts status - one bit per cause:
2122 + * 0 = no interrupt pending
2123 + * 1 = interrupt pending
2124 + *
2125 + * Return: '0' on Success; Error code otherwise.
2126 + */
2127 +int dpdmai_get_irq_status(struct fsl_mc_io *mc_io,
2128 + uint32_t cmd_flags,
2129 + uint16_t token,
2130 + uint8_t irq_index,
2131 + uint32_t *status);
2132 +
2133 +/**
2134 + * dpdmai_clear_irq_status() - Clear a pending interrupt's status
2135 + * @mc_io: Pointer to MC portal's I/O object
2136 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2137 + * @token: Token of DPDMAI object
2138 + * @irq_index: The interrupt index to configure
2139 + * @status: bits to clear (W1C) - one bit per cause:
2140 + * 0 = don't change
2141 + * 1 = clear status bit
2142 + *
2143 + * Return: '0' on Success; Error code otherwise.
2144 + */
2145 +int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io,
2146 + uint32_t cmd_flags,
2147 + uint16_t token,
2148 + uint8_t irq_index,
2149 + uint32_t status);
2150 +
2151 +/**
2152 + * struct dpdmai_attr - Structure representing DPDMAI attributes
2153 + * @id: DPDMAI object ID
2154 + * @version: DPDMAI version
2155 + * @num_of_priorities: number of priorities
2156 + */
2157 +struct dpdmai_attr {
2158 + int id;
2159 + /**
2160 + * struct version - DPDMAI version
2161 + * @major: DPDMAI major version
2162 + * @minor: DPDMAI minor version
2163 + */
2164 + struct {
2165 + uint16_t major;
2166 + uint16_t minor;
2167 + } version;
2168 + uint8_t num_of_priorities;
2169 +};
2170 +
2171 +/**
2172 + * dpdmai_get_attributes() - Retrieve DPDMAI attributes.
2173 + * @mc_io: Pointer to MC portal's I/O object
2174 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2175 + * @token: Token of DPDMAI object
2176 + * @attr: Returned object's attributes
2177 + *
2178 + * Return: '0' on Success; Error code otherwise.
2179 + */
2180 +int dpdmai_get_attributes(struct fsl_mc_io *mc_io,
2181 + uint32_t cmd_flags,
2182 + uint16_t token,
2183 + struct dpdmai_attr *attr);
2184 +
2185 +/**
2186 + * enum dpdmai_dest - DPDMAI destination types
2187 + * @DPDMAI_DEST_NONE: Unassigned destination; The queue is set in parked mode
2188 + * and does not generate FQDAN notifications; user is expected to dequeue
2189 + * from the queue based on polling or other user-defined method
2190 + * @DPDMAI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN
2191 + * notifications to the specified DPIO; user is expected to dequeue
2192 + * from the queue only after notification is received
2193 + * @DPDMAI_DEST_DPCON: The queue is set in schedule mode and does not generate
2194 + * FQDAN notifications, but is connected to the specified DPCON object;
2195 + * user is expected to dequeue from the DPCON channel
2196 + */
2197 +enum dpdmai_dest {
2198 + DPDMAI_DEST_NONE = 0,
2199 + DPDMAI_DEST_DPIO = 1,
2200 + DPDMAI_DEST_DPCON = 2
2201 +};
2202 +
2203 +/**
2204 + * struct dpdmai_dest_cfg - Structure representing DPDMAI destination parameters
2205 + * @dest_type: Destination type
2206 + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type
2207 + * @priority: Priority selection within the DPIO or DPCON channel; valid values
2208 + * are 0-1 or 0-7, depending on the number of priorities in that
2209 + * channel; not relevant for 'DPDMAI_DEST_NONE' option
2210 + */
2211 +struct dpdmai_dest_cfg {
2212 + enum dpdmai_dest dest_type;
2213 + int dest_id;
2214 + uint8_t priority;
2215 +};
2216 +
2217 +/* DPDMAI queue modification options */
2218 +
2219 +/**
2220 + * Select to modify the user's context associated with the queue
2221 + */
2222 +#define DPDMAI_QUEUE_OPT_USER_CTX 0x00000001
2223 +
2224 +/**
2225 + * Select to modify the queue's destination
2226 + */
2227 +#define DPDMAI_QUEUE_OPT_DEST 0x00000002
2228 +
2229 +/**
2230 + * struct dpdmai_rx_queue_cfg - DPDMAI RX queue configuration
2231 + * @options: Flags representing the suggested modifications to the queue;
2232 + * Use any combination of 'DPDMAI_QUEUE_OPT_<X>' flags
2233 + * @user_ctx: User context value provided in the frame descriptor of each
2234 + * dequeued frame;
2235 + * valid only if 'DPDMAI_QUEUE_OPT_USER_CTX' is contained in 'options'
2236 + * @dest_cfg: Queue destination parameters;
2237 + * valid only if 'DPDMAI_QUEUE_OPT_DEST' is contained in 'options'
2238 + */
2239 +struct dpdmai_rx_queue_cfg {
2240 + uint32_t options;
2241 + uint64_t user_ctx;
2242 + struct dpdmai_dest_cfg dest_cfg;
2243 +
2244 +};
2245 +
2246 +/**
2247 + * dpdmai_set_rx_queue() - Set Rx queue configuration
2248 + * @mc_io: Pointer to MC portal's I/O object
2249 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2250 + * @token: Token of DPDMAI object
2251 + * @priority: Select the queue relative to number of
2252 + * priorities configured at DPDMAI creation; use
2253 + * DPDMAI_ALL_QUEUES to configure all Rx queues
2254 + * identically.
2255 + * @cfg: Rx queue configuration
2256 + *
2257 + * Return: '0' on Success; Error code otherwise.
2258 + */
2259 +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io,
2260 + uint32_t cmd_flags,
2261 + uint16_t token,
2262 + uint8_t priority,
2263 + const struct dpdmai_rx_queue_cfg *cfg);
2264 +
2265 +/**
2266 + * struct dpdmai_rx_queue_attr - Structure representing attributes of Rx queues
2267 + * @user_ctx: User context value provided in the frame descriptor of each
2268 + * dequeued frame
2269 + * @dest_cfg: Queue destination configuration
2270 + * @fqid: Virtual FQID value to be used for dequeue operations
2271 + */
2272 +struct dpdmai_rx_queue_attr {
2273 + uint64_t user_ctx;
2274 + struct dpdmai_dest_cfg dest_cfg;
2275 + uint32_t fqid;
2276 +};
2277 +
2278 +/**
2279 + * dpdmai_get_rx_queue() - Retrieve Rx queue attributes.
2280 + * @mc_io: Pointer to MC portal's I/O object
2281 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2282 + * @token: Token of DPDMAI object
2283 + * @priority: Select the queue relative to number of
2284 + * priorities configured at DPDMAI creation
2285 + * @attr: Returned Rx queue attributes
2286 + *
2287 + * Return: '0' on Success; Error code otherwise.
2288 + */
2289 +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io,
2290 + uint32_t cmd_flags,
2291 + uint16_t token,
2292 + uint8_t priority,
2293 + struct dpdmai_rx_queue_attr *attr);
2294 +
2295 +/**
2296 + * struct dpdmai_tx_queue_attr - Structure representing attributes of Tx queues
2297 + * @fqid: Virtual FQID to be used for sending frames to DMA hardware
2298 + */
2299 +
2300 +struct dpdmai_tx_queue_attr {
2301 + uint32_t fqid;
2302 +};
2303 +
2304 +/**
2305 + * dpdmai_get_tx_queue() - Retrieve Tx queue attributes.
2306 + * @mc_io: Pointer to MC portal's I/O object
2307 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'
2308 + * @token: Token of DPDMAI object
2309 + * @priority: Select the queue relative to number of
2310 + * priorities configured at DPDMAI creation
2311 + * @attr: Returned Tx queue attributes
2312 + *
2313 + * Return: '0' on Success; Error code otherwise.
2314 + */
2315 +int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io,
2316 + uint32_t cmd_flags,
2317 + uint16_t token,
2318 + uint8_t priority,
2319 + struct dpdmai_tx_queue_attr *attr);
2320 +
2321 +#endif /* __FSL_DPDMAI_H */
2322 --- /dev/null
2323 +++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h
2324 @@ -0,0 +1,222 @@
2325 +/* Copyright 2013-2016 Freescale Semiconductor Inc.
2326 + *
2327 + * Redistribution and use in source and binary forms, with or without
2328 + * modification, are permitted provided that the following conditions are met:
2329 + * * Redistributions of source code must retain the above copyright
2330 + * notice, this list of conditions and the following disclaimer.
2331 + * * Redistributions in binary form must reproduce the above copyright
2332 + * notice, this list of conditions and the following disclaimer in the
2333 + * documentation and/or other materials provided with the distribution.
2334 + * * Neither the name of the above-listed copyright holders nor the
2335 + * names of any contributors may be used to endorse or promote products
2336 + * derived from this software without specific prior written permission.
2337 + *
2338 + *
2339 + * ALTERNATIVELY, this software may be distributed under the terms of the
2340 + * GNU General Public License ("GPL") as published by the Free Software
2341 + * Foundation, either version 2 of that License or (at your option) any
2342 + * later version.
2343 + *
2344 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
2345 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2346 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2347 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
2348 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
2349 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
2350 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
2351 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
2352 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
2353 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
2354 + * POSSIBILITY OF SUCH DAMAGE.
2355 + */
2356 +#ifndef _FSL_DPDMAI_CMD_H
2357 +#define _FSL_DPDMAI_CMD_H
2358 +
2359 +/* DPDMAI Version */
2360 +#define DPDMAI_VER_MAJOR 2
2361 +#define DPDMAI_VER_MINOR 2
2362 +
2363 +#define DPDMAI_CMD_BASE_VERSION 0
2364 +#define DPDMAI_CMD_ID_OFFSET 4
2365 +
2366 +/* Command IDs */
2367 +#define DPDMAI_CMDID_CLOSE ((0x800 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2368 +#define DPDMAI_CMDID_OPEN ((0x80E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2369 +#define DPDMAI_CMDID_CREATE ((0x90E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2370 +#define DPDMAI_CMDID_DESTROY ((0x900 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2371 +
2372 +#define DPDMAI_CMDID_ENABLE ((0x002 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2373 +#define DPDMAI_CMDID_DISABLE ((0x003 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2374 +#define DPDMAI_CMDID_GET_ATTR ((0x004 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2375 +#define DPDMAI_CMDID_RESET ((0x005 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2376 +#define DPDMAI_CMDID_IS_ENABLED ((0x006 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2377 +
2378 +#define DPDMAI_CMDID_SET_IRQ ((0x010 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2379 +#define DPDMAI_CMDID_GET_IRQ ((0x011 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2380 +#define DPDMAI_CMDID_SET_IRQ_ENABLE ((0x012 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2381 +#define DPDMAI_CMDID_GET_IRQ_ENABLE ((0x013 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2382 +#define DPDMAI_CMDID_SET_IRQ_MASK ((0x014 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2383 +#define DPDMAI_CMDID_GET_IRQ_MASK ((0x015 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2384 +#define DPDMAI_CMDID_GET_IRQ_STATUS ((0x016 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2385 +#define DPDMAI_CMDID_CLEAR_IRQ_STATUS ((0x017 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2386 +
2387 +#define DPDMAI_CMDID_SET_RX_QUEUE ((0x1A0 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2388 +#define DPDMAI_CMDID_GET_RX_QUEUE ((0x1A1 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2389 +#define DPDMAI_CMDID_GET_TX_QUEUE ((0x1A2 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION)
2390 +
2391 +
2392 +#define MC_CMD_HDR_TOKEN_O 32 /* Token field offset */
2393 +#define MC_CMD_HDR_TOKEN_S 16 /* Token field size */
2394 +
2395 +
2396 +#define MAKE_UMASK64(_width) \
2397 + ((uint64_t)((_width) < 64 ? ((uint64_t)1 << (_width)) - 1 : \
2398 + (uint64_t)-1))
2399 +
2400 +static inline uint64_t mc_enc(int lsoffset, int width, uint64_t val)
2401 +{
2402 + return (uint64_t)(((uint64_t)val & MAKE_UMASK64(width)) << lsoffset);
2403 +}
2404 +
2405 +static inline uint64_t mc_dec(uint64_t val, int lsoffset, int width)
2406 +{
2407 + return (uint64_t)((val >> lsoffset) & MAKE_UMASK64(width));
2408 +}
2409 +
2410 +#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \
2411 + ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg))
2412 +
2413 +#define MC_RSP_OP(_cmd, _param, _offset, _width, _type, _arg) \
2414 + (_arg = (_type)mc_dec(_cmd.params[_param], (_offset), (_width)))
2415 +
2416 +#define MC_CMD_HDR_READ_TOKEN(_hdr) \
2417 + ((uint16_t)mc_dec((_hdr), MC_CMD_HDR_TOKEN_O, MC_CMD_HDR_TOKEN_S))
2418 +
2419 +/* cmd, param, offset, width, type, arg_name */
2420 +#define DPDMAI_CMD_OPEN(cmd, dpdmai_id) \
2421 + MC_CMD_OP(cmd, 0, 0, 32, int, dpdmai_id)
2422 +
2423 +/* cmd, param, offset, width, type, arg_name */
2424 +#define DPDMAI_CMD_CREATE(cmd, cfg) \
2425 +do { \
2426 + MC_CMD_OP(cmd, 0, 8, 8, uint8_t, cfg->priorities[0]);\
2427 + MC_CMD_OP(cmd, 0, 16, 8, uint8_t, cfg->priorities[1]);\
2428 +} while (0)
2429 +
2430 +/* cmd, param, offset, width, type, arg_name */
2431 +#define DPDMAI_RSP_IS_ENABLED(cmd, en) \
2432 + MC_RSP_OP(cmd, 0, 0, 1, int, en)
2433 +
2434 +/* cmd, param, offset, width, type, arg_name */
2435 +#define DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg) \
2436 +do { \
2437 + MC_CMD_OP(cmd, 0, 0, 8, uint8_t, irq_index);\
2438 + MC_CMD_OP(cmd, 0, 32, 32, uint32_t, irq_cfg->val);\
2439 + MC_CMD_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\
2440 + MC_CMD_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \
2441 +} while (0)
2442 +
2443 +/* cmd, param, offset, width, type, arg_name */
2444 +#define DPDMAI_CMD_GET_IRQ(cmd, irq_index) \
2445 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index)
2446 +
2447 +/* cmd, param, offset, width, type, arg_name */
2448 +#define DPDMAI_RSP_GET_IRQ(cmd, type, irq_cfg) \
2449 +do { \
2450 + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, irq_cfg->val); \
2451 + MC_RSP_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\
2452 + MC_RSP_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \
2453 + MC_RSP_OP(cmd, 2, 32, 32, int, type); \
2454 +} while (0)
2455 +
2456 +/* cmd, param, offset, width, type, arg_name */
2457 +#define DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, enable_state) \
2458 +do { \
2459 + MC_CMD_OP(cmd, 0, 0, 8, uint8_t, enable_state); \
2460 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \
2461 +} while (0)
2462 +
2463 +/* cmd, param, offset, width, type, arg_name */
2464 +#define DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index) \
2465 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index)
2466 +
2467 +/* cmd, param, offset, width, type, arg_name */
2468 +#define DPDMAI_RSP_GET_IRQ_ENABLE(cmd, enable_state) \
2469 + MC_RSP_OP(cmd, 0, 0, 8, uint8_t, enable_state)
2470 +
2471 +/* cmd, param, offset, width, type, arg_name */
2472 +#define DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask) \
2473 +do { \
2474 + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, mask); \
2475 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \
2476 +} while (0)
2477 +
2478 +/* cmd, param, offset, width, type, arg_name */
2479 +#define DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index) \
2480 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index)
2481 +
2482 +/* cmd, param, offset, width, type, arg_name */
2483 +#define DPDMAI_RSP_GET_IRQ_MASK(cmd, mask) \
2484 + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, mask)
2485 +
2486 +/* cmd, param, offset, width, type, arg_name */
2487 +#define DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, status) \
2488 +do { \
2489 + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status);\
2490 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\
2491 +} while (0)
2492 +
2493 +/* cmd, param, offset, width, type, arg_name */
2494 +#define DPDMAI_RSP_GET_IRQ_STATUS(cmd, status) \
2495 + MC_RSP_OP(cmd, 0, 0, 32, uint32_t, status)
2496 +
2497 +/* cmd, param, offset, width, type, arg_name */
2498 +#define DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status) \
2499 +do { \
2500 + MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status); \
2501 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \
2502 +} while (0)
2503 +
2504 +/* cmd, param, offset, width, type, arg_name */
2505 +#define DPDMAI_RSP_GET_ATTR(cmd, attr) \
2506 +do { \
2507 + MC_RSP_OP(cmd, 0, 0, 32, int, attr->id); \
2508 + MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->num_of_priorities); \
2509 + MC_RSP_OP(cmd, 1, 0, 16, uint16_t, attr->version.major);\
2510 + MC_RSP_OP(cmd, 1, 16, 16, uint16_t, attr->version.minor);\
2511 +} while (0)
2512 +
2513 +/* cmd, param, offset, width, type, arg_name */
2514 +#define DPDMAI_CMD_SET_RX_QUEUE(cmd, priority, cfg) \
2515 +do { \
2516 + MC_CMD_OP(cmd, 0, 0, 32, int, cfg->dest_cfg.dest_id); \
2517 + MC_CMD_OP(cmd, 0, 32, 8, uint8_t, cfg->dest_cfg.priority); \
2518 + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority); \
2519 + MC_CMD_OP(cmd, 0, 48, 4, enum dpdmai_dest, cfg->dest_cfg.dest_type); \
2520 + MC_CMD_OP(cmd, 1, 0, 64, uint64_t, cfg->user_ctx); \
2521 + MC_CMD_OP(cmd, 2, 0, 32, uint32_t, cfg->options);\
2522 +} while (0)
2523 +
2524 +/* cmd, param, offset, width, type, arg_name */
2525 +#define DPDMAI_CMD_GET_RX_QUEUE(cmd, priority) \
2526 + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority)
2527 +
2528 +/* cmd, param, offset, width, type, arg_name */
2529 +#define DPDMAI_RSP_GET_RX_QUEUE(cmd, attr) \
2530 +do { \
2531 + MC_RSP_OP(cmd, 0, 0, 32, int, attr->dest_cfg.dest_id);\
2532 + MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->dest_cfg.priority);\
2533 + MC_RSP_OP(cmd, 0, 48, 4, enum dpdmai_dest, attr->dest_cfg.dest_type);\
2534 + MC_RSP_OP(cmd, 1, 0, 64, uint64_t, attr->user_ctx);\
2535 + MC_RSP_OP(cmd, 2, 0, 32, uint32_t, attr->fqid);\
2536 +} while (0)
2537 +
2538 +/* cmd, param, offset, width, type, arg_name */
2539 +#define DPDMAI_CMD_GET_TX_QUEUE(cmd, priority) \
2540 + MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority)
2541 +
2542 +/* cmd, param, offset, width, type, arg_name */
2543 +#define DPDMAI_RSP_GET_TX_QUEUE(cmd, attr) \
2544 + MC_RSP_OP(cmd, 1, 0, 32, uint32_t, attr->fqid)
2545 +
2546 +#endif /* _FSL_DPDMAI_CMD_H */
2547 --- /dev/null
2548 +++ b/drivers/dma/fsl-qdma.c
2549 @@ -0,0 +1,1201 @@
2550 +/*
2551 + * drivers/dma/fsl-qdma.c
2552 + *
2553 + * Copyright 2014-2015 Freescale Semiconductor, Inc.
2554 + *
2555 + * Driver for the Freescale qDMA engine with software command queue mode.
2556 + * Channel virtualization is supported through enqueuing of DMA jobs to,
2557 + * or dequeuing DMA jobs from, different work queues.
2558 + * This module can be found on Freescale LS SoCs.
2559 + *
2560 + * This program is free software; you can redistribute it and/or modify it
2561 + * under the terms of the GNU General Public License as published by the
2562 + * Free Software Foundation; either version 2 of the License, or (at your
2563 + * option) any later version.
2564 + */
2565 +
2566 +#include <asm/cacheflush.h>
2567 +#include <linux/clk.h>
2568 +#include <linux/delay.h>
2569 +#include <linux/dma-mapping.h>
2570 +#include <linux/dmapool.h>
2571 +#include <linux/init.h>
2572 +#include <linux/interrupt.h>
2573 +#include <linux/module.h>
2574 +#include <linux/of.h>
2575 +#include <linux/of_address.h>
2576 +#include <linux/of_device.h>
2577 +#include <linux/of_dma.h>
2578 +#include <linux/of_irq.h>
2579 +#include <linux/slab.h>
2580 +#include <linux/spinlock.h>
2581 +
2582 +#include "virt-dma.h"
2583 +
2584 +#define FSL_QDMA_DMR 0x0
2585 +#define FSL_QDMA_DSR 0x4
2586 +#define FSL_QDMA_DEIER 0xe00
2587 +#define FSL_QDMA_DEDR 0xe04
2588 +#define FSL_QDMA_DECFDW0R 0xe10
2589 +#define FSL_QDMA_DECFDW1R 0xe14
2590 +#define FSL_QDMA_DECFDW2R 0xe18
2591 +#define FSL_QDMA_DECFDW3R 0xe1c
2592 +#define FSL_QDMA_DECFQIDR 0xe30
2593 +#define FSL_QDMA_DECBR 0xe34
2594 +
2595 +#define FSL_QDMA_BCQMR(x) (0xc0 + 0x100 * (x))
2596 +#define FSL_QDMA_BCQSR(x) (0xc4 + 0x100 * (x))
2597 +#define FSL_QDMA_BCQEDPA_SADDR(x) (0xc8 + 0x100 * (x))
2598 +#define FSL_QDMA_BCQDPA_SADDR(x) (0xcc + 0x100 * (x))
2599 +#define FSL_QDMA_BCQEEPA_SADDR(x) (0xd0 + 0x100 * (x))
2600 +#define FSL_QDMA_BCQEPA_SADDR(x) (0xd4 + 0x100 * (x))
2601 +#define FSL_QDMA_BCQIER(x) (0xe0 + 0x100 * (x))
2602 +#define FSL_QDMA_BCQIDR(x) (0xe4 + 0x100 * (x))
2603 +
2604 +#define FSL_QDMA_SQDPAR 0x80c
2605 +#define FSL_QDMA_SQEPAR 0x814
2606 +#define FSL_QDMA_BSQMR 0x800
2607 +#define FSL_QDMA_BSQSR 0x804
2608 +#define FSL_QDMA_BSQICR 0x828
2609 +#define FSL_QDMA_CQMR 0xa00
2610 +#define FSL_QDMA_CQDSCR1 0xa08
2611 +#define FSL_QDMA_CQDSCR2 0xa0c
2612 +#define FSL_QDMA_CQIER 0xa10
2613 +#define FSL_QDMA_CQEDR 0xa14
2614 +#define FSL_QDMA_SQCCMR 0xa20
2615 +
2616 +#define FSL_QDMA_SQICR_ICEN
2617 +
2618 +#define FSL_QDMA_CQIDR_CQT 0xff000000
2619 +#define FSL_QDMA_CQIDR_SQPE 0x800000
2620 +#define FSL_QDMA_CQIDR_SQT 0x8000
2621 +
2622 +#define FSL_QDMA_BCQIER_CQTIE 0x8000
2623 +#define FSL_QDMA_BCQIER_CQPEIE 0x800000
2624 +#define FSL_QDMA_BSQICR_ICEN 0x80000000
2625 +#define FSL_QDMA_BSQICR_ICST(x) ((x) << 16)
2626 +#define FSL_QDMA_CQIER_MEIE 0x80000000
2627 +#define FSL_QDMA_CQIER_TEIE 0x1
2628 +#define FSL_QDMA_SQCCMR_ENTER_WM 0x200000
2629 +
2630 +#define FSL_QDMA_QUEUE_MAX 8
2631 +
2632 +#define FSL_QDMA_BCQMR_EN 0x80000000
2633 +#define FSL_QDMA_BCQMR_EI 0x40000000
2634 +#define FSL_QDMA_BCQMR_CD_THLD(x) ((x) << 20)
2635 +#define FSL_QDMA_BCQMR_CQ_SIZE(x) ((x) << 16)
2636 +
2637 +#define FSL_QDMA_BCQSR_QF 0x10000
2638 +#define FSL_QDMA_BCQSR_XOFF 0x1
2639 +
2640 +#define FSL_QDMA_BSQMR_EN 0x80000000
2641 +#define FSL_QDMA_BSQMR_DI 0x40000000
2642 +#define FSL_QDMA_BSQMR_CQ_SIZE(x) ((x) << 16)
2643 +
2644 +#define FSL_QDMA_BSQSR_QE 0x20000
2645 +
2646 +#define FSL_QDMA_DMR_DQD 0x40000000
2647 +#define FSL_QDMA_DSR_DB 0x80000000
2648 +
2649 +#define FSL_QDMA_BASE_BUFFER_SIZE 96
2650 +#define FSL_QDMA_EXPECT_SG_ENTRY_NUM 16
2651 +#define FSL_QDMA_CIRCULAR_DESC_SIZE_MIN 64
2652 +#define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX 16384
2653 +#define FSL_QDMA_QUEUE_NUM_MAX 8
2654 +
2655 +#define FSL_QDMA_CMD_RWTTYPE 0x4
2656 +#define FSL_QDMA_CMD_LWC 0x2
2657 +
2658 +#define FSL_QDMA_CMD_RWTTYPE_OFFSET 28
2659 +#define FSL_QDMA_CMD_NS_OFFSET 27
2660 +#define FSL_QDMA_CMD_DQOS_OFFSET 24
2661 +#define FSL_QDMA_CMD_WTHROTL_OFFSET 20
2662 +#define FSL_QDMA_CMD_DSEN_OFFSET 19
2663 +#define FSL_QDMA_CMD_LWC_OFFSET 16
2664 +
2665 +#define FSL_QDMA_E_SG_TABLE 1
2666 +#define FSL_QDMA_E_DATA_BUFFER 0
2667 +#define FSL_QDMA_F_LAST_ENTRY 1
2668 +
2669 +u64 pre_addr, pre_queue;
2670 +
2671 +struct fsl_qdma_ccdf {
2672 + u8 status;
2673 + u32 rev1:22;
2674 + u32 ser:1;
2675 + u32 rev2:1;
2676 + u32 rev3:20;
2677 + u32 offset:9;
2678 + u32 format:3;
2679 + union {
2680 + struct {
2681 + u32 addr_lo; /* low 32-bits of 40-bit address */
2682 + u32 addr_hi:8; /* high 8-bits of 40-bit address */
2683 + u32 rev4:16;
2684 + u32 queue:3;
2685 + u32 rev5:3;
2686 + u32 dd:2; /* dynamic debug */
2687 + };
2688 + struct {
2689 + u64 addr:40;
2690 + /* More efficient address accessor */
2691 + u64 __notaddress:24;
2692 + };
2693 + };
2694 +} __packed;
2695 +
2696 +struct fsl_qdma_csgf {
2697 + u32 offset:13;
2698 + u32 rev1:19;
2699 + u32 length:30;
2700 + u32 f:1;
2701 + u32 e:1;
2702 + union {
2703 + struct {
2704 + u32 addr_lo; /* low 32-bits of 40-bit address */
2705 + u32 addr_hi:8; /* high 8-bits of 40-bit address */
2706 + u32 rev2:24;
2707 + };
2708 + struct {
2709 + u64 addr:40;
2710 + /* More efficient address accessor */
2711 + u64 __notaddress:24;
2712 + };
2713 + };
2714 +} __packed;
2715 +
2716 +struct fsl_qdma_sdf {
2717 + u32 rev3:32;
2718 + u32 ssd:12; /* souce stride distance */
2719 + u32 sss:12; /* souce stride size */
2720 + u32 rev4:8;
2721 + u32 rev5:32;
2722 + u32 cmd;
2723 +} __packed;
2724 +
2725 +struct fsl_qdma_ddf {
2726 + u32 rev1:32;
2727 + u32 dsd:12; /* Destination stride distance */
2728 + u32 dss:12; /* Destination stride size */
2729 + u32 rev2:8;
2730 + u32 rev3:32;
2731 + u32 cmd;
2732 +} __packed;
2733 +
2734 +struct fsl_qdma_chan {
2735 + struct virt_dma_chan vchan;
2736 + struct virt_dma_desc vdesc;
2737 + enum dma_status status;
2738 + u32 slave_id;
2739 + struct fsl_qdma_engine *qdma;
2740 + struct fsl_qdma_queue *queue;
2741 + struct list_head qcomp;
2742 +};
2743 +
2744 +struct fsl_qdma_queue {
2745 + struct fsl_qdma_ccdf *virt_head;
2746 + struct fsl_qdma_ccdf *virt_tail;
2747 + struct list_head comp_used;
2748 + struct list_head comp_free;
2749 + struct dma_pool *comp_pool;
2750 + struct dma_pool *sg_pool;
2751 + spinlock_t queue_lock;
2752 + dma_addr_t bus_addr;
2753 + u32 n_cq;
2754 + u32 id;
2755 + struct fsl_qdma_ccdf *cq;
2756 +};
2757 +
2758 +struct fsl_qdma_sg {
2759 + dma_addr_t bus_addr;
2760 + void *virt_addr;
2761 +};
2762 +
2763 +struct fsl_qdma_comp {
2764 + dma_addr_t bus_addr;
2765 + void *virt_addr;
2766 + struct fsl_qdma_chan *qchan;
2767 + struct fsl_qdma_sg *sg_block;
2768 + struct virt_dma_desc vdesc;
2769 + struct list_head list;
2770 + u32 sg_block_src;
2771 + u32 sg_block_dst;
2772 +};
2773 +
2774 +struct fsl_qdma_engine {
2775 + struct dma_device dma_dev;
2776 + void __iomem *ctrl_base;
2777 + void __iomem *status_base;
2778 + void __iomem *block_base;
2779 + u32 n_chans;
2780 + u32 n_queues;
2781 + struct mutex fsl_qdma_mutex;
2782 + int error_irq;
2783 + int queue_irq;
2784 + bool big_endian;
2785 + struct fsl_qdma_queue *queue;
2786 + struct fsl_qdma_queue *status;
2787 + struct fsl_qdma_chan chans[];
2788 +
2789 +};
2790 +
2791 +static u32 qdma_readl(struct fsl_qdma_engine *qdma, void __iomem *addr)
2792 +{
2793 + if (qdma->big_endian)
2794 + return ioread32be(addr);
2795 + else
2796 + return ioread32(addr);
2797 +}
2798 +
2799 +static void qdma_writel(struct fsl_qdma_engine *qdma, u32 val,
2800 + void __iomem *addr)
2801 +{
2802 + if (qdma->big_endian)
2803 + iowrite32be(val, addr);
2804 + else
2805 + iowrite32(val, addr);
2806 +}
2807 +
2808 +static struct fsl_qdma_chan *to_fsl_qdma_chan(struct dma_chan *chan)
2809 +{
2810 + return container_of(chan, struct fsl_qdma_chan, vchan.chan);
2811 +}
2812 +
2813 +static struct fsl_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd)
2814 +{
2815 + return container_of(vd, struct fsl_qdma_comp, vdesc);
2816 +}
2817 +
2818 +static int fsl_qdma_alloc_chan_resources(struct dma_chan *chan)
2819 +{
2820 + /*
2821 + * In QDMA mode, We don't need to do anything.
2822 + */
2823 + return 0;
2824 +}
2825 +
2826 +static void fsl_qdma_free_chan_resources(struct dma_chan *chan)
2827 +{
2828 + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
2829 + unsigned long flags;
2830 + LIST_HEAD(head);
2831 +
2832 + spin_lock_irqsave(&fsl_chan->vchan.lock, flags);
2833 + vchan_get_all_descriptors(&fsl_chan->vchan, &head);
2834 + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);
2835 +
2836 + vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
2837 +}
2838 +
2839 +static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,
2840 + dma_addr_t dst, dma_addr_t src, u32 len)
2841 +{
2842 + struct fsl_qdma_ccdf *ccdf;
2843 + struct fsl_qdma_csgf *csgf_desc, *csgf_src, *csgf_dest;
2844 + struct fsl_qdma_sdf *sdf;
2845 + struct fsl_qdma_ddf *ddf;
2846 +
2847 + ccdf = (struct fsl_qdma_ccdf *)fsl_comp->virt_addr;
2848 + csgf_desc = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 1;
2849 + csgf_src = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 2;
2850 + csgf_dest = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 3;
2851 + sdf = (struct fsl_qdma_sdf *)fsl_comp->virt_addr + 4;
2852 + ddf = (struct fsl_qdma_ddf *)fsl_comp->virt_addr + 5;
2853 +
2854 + memset(fsl_comp->virt_addr, 0, FSL_QDMA_BASE_BUFFER_SIZE);
2855 + /* Head Command Descriptor(Frame Descriptor) */
2856 + ccdf->addr = fsl_comp->bus_addr + 16;
2857 + ccdf->format = 1; /* Compound S/G format */
2858 + /* Status notification is enqueued to status queue. */
2859 + ccdf->ser = 1;
2860 + /* Compound Command Descriptor(Frame List Table) */
2861 + csgf_desc->addr = fsl_comp->bus_addr + 64;
2862 + /* It must be 32 as Compound S/G Descriptor */
2863 + csgf_desc->length = 32;
2864 + csgf_src->addr = src;
2865 + csgf_src->length = len;
2866 + csgf_dest->addr = dst;
2867 + csgf_dest->length = len;
2868 + /* This entry is the last entry. */
2869 + csgf_dest->f = FSL_QDMA_F_LAST_ENTRY;
2870 + /* Descriptor Buffer */
2871 + sdf->cmd = FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET;
2872 + ddf->cmd = FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET;
2873 + ddf->cmd |= FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET;
2874 +}
2875 +
2876 +static void fsl_qdma_comp_fill_sg(
2877 + struct fsl_qdma_comp *fsl_comp,
2878 + struct scatterlist *dst_sg, unsigned int dst_nents,
2879 + struct scatterlist *src_sg, unsigned int src_nents)
2880 +{
2881 + struct fsl_qdma_ccdf *ccdf;
2882 + struct fsl_qdma_csgf *csgf_desc, *csgf_src, *csgf_dest, *csgf_sg;
2883 + struct fsl_qdma_sdf *sdf;
2884 + struct fsl_qdma_ddf *ddf;
2885 + struct fsl_qdma_sg *sg_block, *temp;
2886 + struct scatterlist *sg;
2887 + u64 total_src_len = 0;
2888 + u64 total_dst_len = 0;
2889 + u32 i;
2890 +
2891 + ccdf = (struct fsl_qdma_ccdf *)fsl_comp->virt_addr;
2892 + csgf_desc = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 1;
2893 + csgf_src = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 2;
2894 + csgf_dest = (struct fsl_qdma_csgf *)fsl_comp->virt_addr + 3;
2895 + sdf = (struct fsl_qdma_sdf *)fsl_comp->virt_addr + 4;
2896 + ddf = (struct fsl_qdma_ddf *)fsl_comp->virt_addr + 5;
2897 +
2898 + memset(fsl_comp->virt_addr, 0, FSL_QDMA_BASE_BUFFER_SIZE);
2899 + /* Head Command Descriptor(Frame Descriptor) */
2900 + ccdf->addr = fsl_comp->bus_addr + 16;
2901 + ccdf->format = 1; /* Compound S/G format */
2902 + /* Status notification is enqueued to status queue. */
2903 + ccdf->ser = 1;
2904 +
2905 + /* Compound Command Descriptor(Frame List Table) */
2906 + csgf_desc->addr = fsl_comp->bus_addr + 64;
2907 + /* It must be 32 as Compound S/G Descriptor */
2908 + csgf_desc->length = 32;
2909 +
2910 + sg_block = fsl_comp->sg_block;
2911 + csgf_src->addr = sg_block->bus_addr;
2912 + /* This entry link to the s/g entry. */
2913 + csgf_src->e = FSL_QDMA_E_SG_TABLE;
2914 +
2915 + temp = sg_block + fsl_comp->sg_block_src;
2916 + csgf_dest->addr = temp->bus_addr;
2917 + /* This entry is the last entry. */
2918 + csgf_dest->f = FSL_QDMA_F_LAST_ENTRY;
2919 + /* This entry link to the s/g entry. */
2920 + csgf_dest->e = FSL_QDMA_E_SG_TABLE;
2921 +
2922 + for_each_sg(src_sg, sg, src_nents, i) {
2923 + temp = sg_block + i / (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1);
2924 + csgf_sg = (struct fsl_qdma_csgf *)temp->virt_addr +
2925 + i % (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1);
2926 + csgf_sg->addr = sg_dma_address(sg);
2927 + csgf_sg->length = sg_dma_len(sg);
2928 + total_src_len += sg_dma_len(sg);
2929 +
2930 + if (i == src_nents - 1)
2931 + csgf_sg->f = FSL_QDMA_F_LAST_ENTRY;
2932 + if (i % (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) ==
2933 + FSL_QDMA_EXPECT_SG_ENTRY_NUM - 2) {
2934 + csgf_sg = (struct fsl_qdma_csgf *)temp->virt_addr +
2935 + FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1;
2936 + temp = sg_block +
2937 + i / (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) + 1;
2938 + csgf_sg->addr = temp->bus_addr;
2939 + csgf_sg->e = FSL_QDMA_E_SG_TABLE;
2940 + }
2941 + }
2942 +
2943 + sg_block += fsl_comp->sg_block_src;
2944 + for_each_sg(dst_sg, sg, dst_nents, i) {
2945 + temp = sg_block + i / (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1);
2946 + csgf_sg = (struct fsl_qdma_csgf *)temp->virt_addr +
2947 + i % (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1);
2948 + csgf_sg->addr = sg_dma_address(sg);
2949 + csgf_sg->length = sg_dma_len(sg);
2950 + total_dst_len += sg_dma_len(sg);
2951 +
2952 + if (i == dst_nents - 1)
2953 + csgf_sg->f = FSL_QDMA_F_LAST_ENTRY;
2954 + if (i % (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) ==
2955 + FSL_QDMA_EXPECT_SG_ENTRY_NUM - 2) {
2956 + csgf_sg = (struct fsl_qdma_csgf *)temp->virt_addr +
2957 + FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1;
2958 + temp = sg_block +
2959 + i / (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) + 1;
2960 + csgf_sg->addr = temp->bus_addr;
2961 + csgf_sg->e = FSL_QDMA_E_SG_TABLE;
2962 + }
2963 + }
2964 +
2965 + if (total_src_len != total_dst_len)
2966 + dev_err(&fsl_comp->qchan->vchan.chan.dev->device,
2967 + "The data length for src and dst isn't match.\n");
2968 +
2969 + csgf_src->length = total_src_len;
2970 + csgf_dest->length = total_dst_len;
2971 +
2972 + /* Descriptor Buffer */
2973 + sdf->cmd = FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET;
2974 + ddf->cmd = FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET;
2975 +}
2976 +
2977 +/*
2978 + * Prei-request full command descriptor for enqueue.
2979 + */
2980 +static int fsl_qdma_pre_request_enqueue_desc(struct fsl_qdma_queue *queue)
2981 +{
2982 + struct fsl_qdma_comp *comp_temp;
2983 + int i;
2984 +
2985 + for (i = 0; i < queue->n_cq; i++) {
2986 + comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
2987 + if (!comp_temp)
2988 + return -1;
2989 + comp_temp->virt_addr = dma_pool_alloc(queue->comp_pool,
2990 + GFP_NOWAIT,
2991 + &comp_temp->bus_addr);
2992 + if (!comp_temp->virt_addr)
2993 + return -1;
2994 + list_add_tail(&comp_temp->list, &queue->comp_free);
2995 + }
2996 + return 0;
2997 +}
2998 +
2999 +/*
3000 + * Request a command descriptor for enqueue.
3001 + */
3002 +static struct fsl_qdma_comp *fsl_qdma_request_enqueue_desc(
3003 + struct fsl_qdma_chan *fsl_chan,
3004 + unsigned int dst_nents,
3005 + unsigned int src_nents)
3006 +{
3007 + struct fsl_qdma_comp *comp_temp;
3008 + struct fsl_qdma_sg *sg_block;
3009 + struct fsl_qdma_queue *queue = fsl_chan->queue;
3010 + unsigned long flags;
3011 + unsigned int dst_sg_entry_block, src_sg_entry_block, sg_entry_total, i;
3012 +
3013 + spin_lock_irqsave(&queue->queue_lock, flags);
3014 + if (list_empty(&queue->comp_free)) {
3015 + spin_unlock_irqrestore(&queue->queue_lock, flags);
3016 + comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL);
3017 + if (!comp_temp)
3018 + return NULL;
3019 + comp_temp->virt_addr = dma_pool_alloc(queue->comp_pool,
3020 + GFP_NOWAIT,
3021 + &comp_temp->bus_addr);
3022 + if (!comp_temp->virt_addr)
3023 + return NULL;
3024 + } else {
3025 + comp_temp = list_first_entry(&queue->comp_free,
3026 + struct fsl_qdma_comp,
3027 + list);
3028 + list_del(&comp_temp->list);
3029 + spin_unlock_irqrestore(&queue->queue_lock, flags);
3030 + }
3031 +
3032 + if (dst_nents != 0)
3033 + dst_sg_entry_block = dst_nents /
3034 + (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) + 1;
3035 + else
3036 + dst_sg_entry_block = 0;
3037 +
3038 + if (src_nents != 0)
3039 + src_sg_entry_block = src_nents /
3040 + (FSL_QDMA_EXPECT_SG_ENTRY_NUM - 1) + 1;
3041 + else
3042 + src_sg_entry_block = 0;
3043 +
3044 + sg_entry_total = dst_sg_entry_block + src_sg_entry_block;
3045 + if (sg_entry_total) {
3046 + sg_block = kzalloc(sizeof(*sg_block) *
3047 + sg_entry_total,
3048 + GFP_KERNEL);
3049 + if (!sg_block)
3050 + return NULL;
3051 + comp_temp->sg_block = sg_block;
3052 + for (i = 0; i < sg_entry_total; i++) {
3053 + sg_block->virt_addr = dma_pool_alloc(queue->sg_pool,
3054 + GFP_NOWAIT,
3055 + &sg_block->bus_addr);
3056 + memset(sg_block->virt_addr, 0,
3057 + FSL_QDMA_EXPECT_SG_ENTRY_NUM * 16);
3058 + sg_block++;
3059 + }
3060 + }
3061 +
3062 + comp_temp->sg_block_src = src_sg_entry_block;
3063 + comp_temp->sg_block_dst = dst_sg_entry_block;
3064 + comp_temp->qchan = fsl_chan;
3065 +
3066 + return comp_temp;
3067 +}
3068 +
3069 +static struct fsl_qdma_queue *fsl_qdma_alloc_queue_resources(
3070 + struct platform_device *pdev,
3071 + unsigned int queue_num)
3072 +{
3073 + struct device_node *np = pdev->dev.of_node;
3074 + struct fsl_qdma_queue *queue_head, *queue_temp;
3075 + int ret, len, i;
3076 + unsigned int queue_size[FSL_QDMA_QUEUE_MAX];
3077 +
3078 + if (queue_num > FSL_QDMA_QUEUE_MAX)
3079 + queue_num = FSL_QDMA_QUEUE_MAX;
3080 + len = sizeof(*queue_head) * queue_num;
3081 + queue_head = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
3082 + if (!queue_head)
3083 + return NULL;
3084 +
3085 + ret = of_property_read_u32_array(np, "queue-sizes", queue_size,
3086 + queue_num);
3087 + if (ret) {
3088 + dev_err(&pdev->dev, "Can't get queue-sizes.\n");
3089 + return NULL;
3090 + }
3091 +
3092 + for (i = 0; i < queue_num; i++) {
3093 + if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX
3094 + || queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
3095 + dev_err(&pdev->dev, "Get wrong queue-sizes.\n");
3096 + return NULL;
3097 + }
3098 + queue_temp = queue_head + i;
3099 + queue_temp->cq = dma_alloc_coherent(&pdev->dev,
3100 + sizeof(struct fsl_qdma_ccdf) *
3101 + queue_size[i],
3102 + &queue_temp->bus_addr,
3103 + GFP_KERNEL);
3104 + if (!queue_temp->cq)
3105 + return NULL;
3106 + queue_temp->n_cq = queue_size[i];
3107 + queue_temp->id = i;
3108 + queue_temp->virt_head = queue_temp->cq;
3109 + queue_temp->virt_tail = queue_temp->cq;
3110 + /*
3111 + * The dma pool for queue command buffer
3112 + */
3113 + queue_temp->comp_pool = dma_pool_create("comp_pool",
3114 + &pdev->dev,
3115 + FSL_QDMA_BASE_BUFFER_SIZE,
3116 + 16, 0);
3117 + if (!queue_temp->comp_pool) {
3118 + dma_free_coherent(&pdev->dev,
3119 + sizeof(struct fsl_qdma_ccdf) *
3120 + queue_size[i],
3121 + queue_temp->cq,
3122 + queue_temp->bus_addr);
3123 + return NULL;
3124 + }
3125 + /*
3126 + * The dma pool for queue command buffer
3127 + */
3128 + queue_temp->sg_pool = dma_pool_create("sg_pool",
3129 + &pdev->dev,
3130 + FSL_QDMA_EXPECT_SG_ENTRY_NUM * 16,
3131 + 64, 0);
3132 + if (!queue_temp->sg_pool) {
3133 + dma_free_coherent(&pdev->dev,
3134 + sizeof(struct fsl_qdma_ccdf) *
3135 + queue_size[i],
3136 + queue_temp->cq,
3137 + queue_temp->bus_addr);
3138 + dma_pool_destroy(queue_temp->comp_pool);
3139 + return NULL;
3140 + }
3141 + /*
3142 + * List for queue command buffer
3143 + */
3144 + INIT_LIST_HEAD(&queue_temp->comp_used);
3145 + INIT_LIST_HEAD(&queue_temp->comp_free);
3146 + spin_lock_init(&queue_temp->queue_lock);
3147 + }
3148 +
3149 + return queue_head;
3150 +}
3151 +
3152 +static struct fsl_qdma_queue *fsl_qdma_prep_status_queue(
3153 + struct platform_device *pdev)
3154 +{
3155 + struct device_node *np = pdev->dev.of_node;
3156 + struct fsl_qdma_queue *status_head;
3157 + unsigned int status_size;
3158 + int ret;
3159 +
3160 + ret = of_property_read_u32(np, "status-sizes", &status_size);
3161 + if (ret) {
3162 + dev_err(&pdev->dev, "Can't get status-sizes.\n");
3163 + return NULL;
3164 + }
3165 + if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX
3166 + || status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
3167 + dev_err(&pdev->dev, "Get wrong status_size.\n");
3168 + return NULL;
3169 + }
3170 + status_head = devm_kzalloc(&pdev->dev, sizeof(*status_head),
3171 + GFP_KERNEL);
3172 + if (!status_head)
3173 + return NULL;
3174 +
3175 + /*
3176 + * Buffer for queue command
3177 + */
3178 + status_head->cq = dma_alloc_coherent(&pdev->dev,
3179 + sizeof(struct fsl_qdma_ccdf) *
3180 + status_size,
3181 + &status_head->bus_addr,
3182 + GFP_KERNEL);
3183 + if (!status_head->cq)
3184 + return NULL;
3185 + status_head->n_cq = status_size;
3186 + status_head->virt_head = status_head->cq;
3187 + status_head->virt_tail = status_head->cq;
3188 + status_head->comp_pool = NULL;
3189 +
3190 + return status_head;
3191 +}
3192 +
3193 +static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma)
3194 +{
3195 + void __iomem *ctrl = fsl_qdma->ctrl_base;
3196 + void __iomem *block = fsl_qdma->block_base;
3197 + int i, count = 5;
3198 + u32 reg;
3199 +
3200 + /* Disable the command queue and wait for idle state. */
3201 + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
3202 + reg |= FSL_QDMA_DMR_DQD;
3203 + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
3204 + for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++)
3205 + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQMR(i));
3206 +
3207 + while (1) {
3208 + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DSR);
3209 + if (!(reg & FSL_QDMA_DSR_DB))
3210 + break;
3211 + if (count-- < 0)
3212 + return -EBUSY;
3213 + udelay(100);
3214 + }
3215 +
3216 + /* Disable status queue. */
3217 + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BSQMR);
3218 +
3219 + /*
3220 + * Clear the command queue interrupt detect register for all queues.
3221 + */
3222 + qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
3223 +
3224 + return 0;
3225 +}
3226 +
3227 +static int fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma)
3228 +{
3229 + struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
3230 + struct fsl_qdma_queue *fsl_status = fsl_qdma->status;
3231 + struct fsl_qdma_queue *temp_queue;
3232 + struct fsl_qdma_comp *fsl_comp;
3233 + struct fsl_qdma_ccdf *status_addr;
3234 + struct fsl_qdma_csgf *csgf_src;
3235 + void __iomem *block = fsl_qdma->block_base;
3236 + u32 reg, i;
3237 + bool duplicate, duplicate_handle;
3238 +
3239 + while (1) {
3240 + duplicate = 0;
3241 + duplicate_handle = 0;
3242 + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQSR);
3243 + if (reg & FSL_QDMA_BSQSR_QE)
3244 + return 0;
3245 + status_addr = fsl_status->virt_head;
3246 + if (status_addr->queue == pre_queue &&
3247 + status_addr->addr == pre_addr)
3248 + duplicate = 1;
3249 +
3250 + i = status_addr->queue;
3251 + pre_queue = status_addr->queue;
3252 + pre_addr = status_addr->addr;
3253 + temp_queue = fsl_queue + i;
3254 + spin_lock(&temp_queue->queue_lock);
3255 + if (list_empty(&temp_queue->comp_used)) {
3256 + if (duplicate)
3257 + duplicate_handle = 1;
3258 + else {
3259 + spin_unlock(&temp_queue->queue_lock);
3260 + return -1;
3261 + }
3262 + } else {
3263 + fsl_comp = list_first_entry(&temp_queue->comp_used,
3264 + struct fsl_qdma_comp,
3265 + list);
3266 + csgf_src = (struct fsl_qdma_csgf *)fsl_comp->virt_addr
3267 + + 2;
3268 + if (fsl_comp->bus_addr + 16 !=
3269 + (dma_addr_t)status_addr->addr) {
3270 + if (duplicate)
3271 + duplicate_handle = 1;
3272 + else {
3273 + spin_unlock(&temp_queue->queue_lock);
3274 + return -1;
3275 + }
3276 + }
3277 + }
3278 +
3279 + if (duplicate_handle) {
3280 + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
3281 + reg |= FSL_QDMA_BSQMR_DI;
3282 + status_addr->addr = 0x0;
3283 + fsl_status->virt_head++;
3284 + if (fsl_status->virt_head == fsl_status->cq
3285 + + fsl_status->n_cq)
3286 + fsl_status->virt_head = fsl_status->cq;
3287 + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
3288 + spin_unlock(&temp_queue->queue_lock);
3289 + continue;
3290 + }
3291 + list_del(&fsl_comp->list);
3292 +
3293 + reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR);
3294 + reg |= FSL_QDMA_BSQMR_DI;
3295 + status_addr->addr = 0x0;
3296 + fsl_status->virt_head++;
3297 + if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq)
3298 + fsl_status->virt_head = fsl_status->cq;
3299 + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
3300 + spin_unlock(&temp_queue->queue_lock);
3301 +
3302 + spin_lock(&fsl_comp->qchan->vchan.lock);
3303 + vchan_cookie_complete(&fsl_comp->vdesc);
3304 + fsl_comp->qchan->status = DMA_COMPLETE;
3305 + spin_unlock(&fsl_comp->qchan->vchan.lock);
3306 + }
3307 + return 0;
3308 +}
3309 +
3310 +static irqreturn_t fsl_qdma_error_handler(int irq, void *dev_id)
3311 +{
3312 + struct fsl_qdma_engine *fsl_qdma = dev_id;
3313 + unsigned int intr;
3314 + void __iomem *status = fsl_qdma->status_base;
3315 +
3316 + intr = qdma_readl(fsl_qdma, status + FSL_QDMA_DEDR);
3317 +
3318 + if (intr)
3319 + dev_err(fsl_qdma->dma_dev.dev, "DMA transaction error!\n");
3320 +
3321 + qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR);
3322 + return IRQ_HANDLED;
3323 +}
3324 +
3325 +static irqreturn_t fsl_qdma_queue_handler(int irq, void *dev_id)
3326 +{
3327 + struct fsl_qdma_engine *fsl_qdma = dev_id;
3328 + unsigned int intr, reg;
3329 + void __iomem *block = fsl_qdma->block_base;
3330 + void __iomem *ctrl = fsl_qdma->ctrl_base;
3331 +
3332 + intr = qdma_readl(fsl_qdma, block + FSL_QDMA_BCQIDR(0));
3333 +
3334 + if ((intr & FSL_QDMA_CQIDR_SQT) != 0)
3335 + intr = fsl_qdma_queue_transfer_complete(fsl_qdma);
3336 +
3337 + if (intr != 0) {
3338 + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
3339 + reg |= FSL_QDMA_DMR_DQD;
3340 + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
3341 + qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQIER(0));
3342 + dev_err(fsl_qdma->dma_dev.dev, "QDMA: status err!\n");
3343 + }
3344 +
3345 + qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
3346 +
3347 + return IRQ_HANDLED;
3348 +}
3349 +
3350 +static int
3351 +fsl_qdma_irq_init(struct platform_device *pdev,
3352 + struct fsl_qdma_engine *fsl_qdma)
3353 +{
3354 + int ret;
3355 +
3356 + fsl_qdma->error_irq = platform_get_irq_byname(pdev,
3357 + "qdma-error");
3358 + if (fsl_qdma->error_irq < 0) {
3359 + dev_err(&pdev->dev, "Can't get qdma controller irq.\n");
3360 + return fsl_qdma->error_irq;
3361 + }
3362 +
3363 + fsl_qdma->queue_irq = platform_get_irq_byname(pdev, "qdma-queue");
3364 + if (fsl_qdma->queue_irq < 0) {
3365 + dev_err(&pdev->dev, "Can't get qdma queue irq.\n");
3366 + return fsl_qdma->queue_irq;
3367 + }
3368 +
3369 + ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq,
3370 + fsl_qdma_error_handler, 0, "qDMA error", fsl_qdma);
3371 + if (ret) {
3372 + dev_err(&pdev->dev, "Can't register qDMA controller IRQ.\n");
3373 + return ret;
3374 + }
3375 + ret = devm_request_irq(&pdev->dev, fsl_qdma->queue_irq,
3376 + fsl_qdma_queue_handler, 0, "qDMA queue", fsl_qdma);
3377 + if (ret) {
3378 + dev_err(&pdev->dev, "Can't register qDMA queue IRQ.\n");
3379 + return ret;
3380 + }
3381 +
3382 + return 0;
3383 +}
3384 +
3385 +static int fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
3386 +{
3387 + struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
3388 + struct fsl_qdma_queue *temp;
3389 + void __iomem *ctrl = fsl_qdma->ctrl_base;
3390 + void __iomem *status = fsl_qdma->status_base;
3391 + void __iomem *block = fsl_qdma->block_base;
3392 + int i, ret;
3393 + u32 reg;
3394 +
3395 + /* Try to halt the qDMA engine first. */
3396 + ret = fsl_qdma_halt(fsl_qdma);
3397 + if (ret) {
3398 + dev_err(fsl_qdma->dma_dev.dev, "DMA halt failed!");
3399 + return ret;
3400 + }
3401 +
3402 + /*
3403 + * Clear the command queue interrupt detect register for all queues.
3404 + */
3405 + qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0));
3406 +
3407 + for (i = 0; i < fsl_qdma->n_queues; i++) {
3408 + temp = fsl_queue + i;
3409 + /*
3410 + * Initialize Command Queue registers to point to the first
3411 + * command descriptor in memory.
3412 + * Dequeue Pointer Address Registers
3413 + * Enqueue Pointer Address Registers
3414 + */
3415 + qdma_writel(fsl_qdma, temp->bus_addr,
3416 + block + FSL_QDMA_BCQDPA_SADDR(i));
3417 + qdma_writel(fsl_qdma, temp->bus_addr,
3418 + block + FSL_QDMA_BCQEPA_SADDR(i));
3419 +
3420 + /* Initialize the queue mode. */
3421 + reg = FSL_QDMA_BCQMR_EN;
3422 + reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq)-4);
3423 + reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq)-6);
3424 + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BCQMR(i));
3425 + }
3426 +
3427 + /*
3428 + * Workaround for erratum: ERR010812.
3429 + * We must enable XOFF to avoid the enqueue rejection occurs.
3430 + * Setting SQCCMR ENTER_WM to 0x20.
3431 + */
3432 + qdma_writel(fsl_qdma, FSL_QDMA_SQCCMR_ENTER_WM,
3433 + block + FSL_QDMA_SQCCMR);
3434 + /*
3435 + * Initialize status queue registers to point to the first
3436 + * command descriptor in memory.
3437 + * Dequeue Pointer Address Registers
3438 + * Enqueue Pointer Address Registers
3439 + */
3440 + qdma_writel(fsl_qdma, fsl_qdma->status->bus_addr,
3441 + block + FSL_QDMA_SQEPAR);
3442 + qdma_writel(fsl_qdma, fsl_qdma->status->bus_addr,
3443 + block + FSL_QDMA_SQDPAR);
3444 + /* Initialize status queue interrupt. */
3445 + qdma_writel(fsl_qdma, FSL_QDMA_BCQIER_CQTIE,
3446 + block + FSL_QDMA_BCQIER(0));
3447 + qdma_writel(fsl_qdma, FSL_QDMA_BSQICR_ICEN | FSL_QDMA_BSQICR_ICST(5)
3448 + | 0x8000,
3449 + block + FSL_QDMA_BSQICR);
3450 + qdma_writel(fsl_qdma, FSL_QDMA_CQIER_MEIE | FSL_QDMA_CQIER_TEIE,
3451 + block + FSL_QDMA_CQIER);
3452 + /* Initialize controller interrupt register. */
3453 + qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR);
3454 + qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEIER);
3455 +
3456 + /* Initialize the status queue mode. */
3457 + reg = FSL_QDMA_BSQMR_EN;
3458 + reg |= FSL_QDMA_BSQMR_CQ_SIZE(ilog2(fsl_qdma->status->n_cq)-6);
3459 + qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR);
3460 +
3461 + reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR);
3462 + reg &= ~FSL_QDMA_DMR_DQD;
3463 + qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR);
3464 +
3465 + return 0;
3466 +}
3467 +
3468 +static struct dma_async_tx_descriptor *fsl_qdma_prep_dma_sg(
3469 + struct dma_chan *chan,
3470 + struct scatterlist *dst_sg, unsigned int dst_nents,
3471 + struct scatterlist *src_sg, unsigned int src_nents,
3472 + unsigned long flags)
3473 +{
3474 + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
3475 + struct fsl_qdma_comp *fsl_comp;
3476 +
3477 + fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan,
3478 + dst_nents,
3479 + src_nents);
3480 + fsl_qdma_comp_fill_sg(fsl_comp, dst_sg, dst_nents, src_sg, src_nents);
3481 +
3482 + return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags);
3483 +}
3484 +
3485 +static struct dma_async_tx_descriptor *
3486 +fsl_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst,
3487 + dma_addr_t src, size_t len, unsigned long flags)
3488 +{
3489 + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
3490 + struct fsl_qdma_comp *fsl_comp;
3491 +
3492 + fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan, 0, 0);
3493 + fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len);
3494 +
3495 + return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags);
3496 +}
3497 +
3498 +static void fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan)
3499 +{
3500 + void __iomem *block = fsl_chan->qdma->block_base;
3501 + struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
3502 + struct fsl_qdma_comp *fsl_comp;
3503 + struct virt_dma_desc *vdesc;
3504 + u32 reg;
3505 +
3506 + reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQSR(fsl_queue->id));
3507 + if (reg & (FSL_QDMA_BCQSR_QF | FSL_QDMA_BCQSR_XOFF))
3508 + return;
3509 + vdesc = vchan_next_desc(&fsl_chan->vchan);
3510 + if (!vdesc)
3511 + return;
3512 + list_del(&vdesc->node);
3513 + fsl_comp = to_fsl_qdma_comp(vdesc);
3514 +
3515 + memcpy(fsl_queue->virt_head++, fsl_comp->virt_addr, 16);
3516 + if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq)
3517 + fsl_queue->virt_head = fsl_queue->cq;
3518 +
3519 + list_add_tail(&fsl_comp->list, &fsl_queue->comp_used);
3520 + barrier();
3521 + reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQMR(fsl_queue->id));
3522 + reg |= FSL_QDMA_BCQMR_EI;
3523 + qdma_writel(fsl_chan->qdma, reg, block + FSL_QDMA_BCQMR(fsl_queue->id));
3524 + fsl_chan->status = DMA_IN_PROGRESS;
3525 +}
3526 +
3527 +static enum dma_status fsl_qdma_tx_status(struct dma_chan *chan,
3528 + dma_cookie_t cookie, struct dma_tx_state *txstate)
3529 +{
3530 + return dma_cookie_status(chan, cookie, txstate);
3531 +}
3532 +
3533 +static void fsl_qdma_free_desc(struct virt_dma_desc *vdesc)
3534 +{
3535 + struct fsl_qdma_comp *fsl_comp;
3536 + struct fsl_qdma_queue *fsl_queue;
3537 + struct fsl_qdma_sg *sg_block;
3538 + unsigned long flags;
3539 + unsigned int i;
3540 +
3541 + fsl_comp = to_fsl_qdma_comp(vdesc);
3542 + fsl_queue = fsl_comp->qchan->queue;
3543 +
3544 + if (fsl_comp->sg_block) {
3545 + for (i = 0; i < fsl_comp->sg_block_src +
3546 + fsl_comp->sg_block_dst; i++) {
3547 + sg_block = fsl_comp->sg_block + i;
3548 + dma_pool_free(fsl_queue->sg_pool,
3549 + sg_block->virt_addr,
3550 + sg_block->bus_addr);
3551 + }
3552 + kfree(fsl_comp->sg_block);
3553 + }
3554 +
3555 + spin_lock_irqsave(&fsl_queue->queue_lock, flags);
3556 + list_add_tail(&fsl_comp->list, &fsl_queue->comp_free);
3557 + spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);
3558 +}
3559 +
3560 +static void fsl_qdma_issue_pending(struct dma_chan *chan)
3561 +{
3562 + struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan);
3563 + struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
3564 + unsigned long flags;
3565 +
3566 + spin_lock_irqsave(&fsl_queue->queue_lock, flags);
3567 + spin_lock(&fsl_chan->vchan.lock);
3568 + if (vchan_issue_pending(&fsl_chan->vchan))
3569 + fsl_qdma_enqueue_desc(fsl_chan);
3570 + spin_unlock(&fsl_chan->vchan.lock);
3571 + spin_unlock_irqrestore(&fsl_queue->queue_lock, flags);
3572 +}
3573 +
3574 +static int fsl_qdma_probe(struct platform_device *pdev)
3575 +{
3576 + struct device_node *np = pdev->dev.of_node;
3577 + struct fsl_qdma_engine *fsl_qdma;
3578 + struct fsl_qdma_chan *fsl_chan;
3579 + struct resource *res;
3580 + unsigned int len, chans, queues;
3581 + int ret, i;
3582 +
3583 + ret = of_property_read_u32(np, "channels", &chans);
3584 + if (ret) {
3585 + dev_err(&pdev->dev, "Can't get channels.\n");
3586 + return ret;
3587 + }
3588 +
3589 + len = sizeof(*fsl_qdma) + sizeof(*fsl_chan) * chans;
3590 + fsl_qdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
3591 + if (!fsl_qdma)
3592 + return -ENOMEM;
3593 +
3594 + ret = of_property_read_u32(np, "queues", &queues);
3595 + if (ret) {
3596 + dev_err(&pdev->dev, "Can't get queues.\n");
3597 + return ret;
3598 + }
3599 +
3600 + fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, queues);
3601 + if (!fsl_qdma->queue)
3602 + return -ENOMEM;
3603 +
3604 + fsl_qdma->status = fsl_qdma_prep_status_queue(pdev);
3605 + if (!fsl_qdma->status)
3606 + return -ENOMEM;
3607 +
3608 + fsl_qdma->n_chans = chans;
3609 + fsl_qdma->n_queues = queues;
3610 + mutex_init(&fsl_qdma->fsl_qdma_mutex);
3611 +
3612 + res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3613 + fsl_qdma->ctrl_base = devm_ioremap_resource(&pdev->dev, res);
3614 + if (IS_ERR(fsl_qdma->ctrl_base))
3615 + return PTR_ERR(fsl_qdma->ctrl_base);
3616 +
3617 + res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
3618 + fsl_qdma->status_base = devm_ioremap_resource(&pdev->dev, res);
3619 + if (IS_ERR(fsl_qdma->status_base))
3620 + return PTR_ERR(fsl_qdma->status_base);
3621 +
3622 + res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
3623 + fsl_qdma->block_base = devm_ioremap_resource(&pdev->dev, res);
3624 + if (IS_ERR(fsl_qdma->block_base))
3625 + return PTR_ERR(fsl_qdma->block_base);
3626 +
3627 + ret = fsl_qdma_irq_init(pdev, fsl_qdma);
3628 + if (ret)
3629 + return ret;
3630 +
3631 + fsl_qdma->big_endian = of_property_read_bool(np, "big-endian");
3632 + INIT_LIST_HEAD(&fsl_qdma->dma_dev.channels);
3633 + for (i = 0; i < fsl_qdma->n_chans; i++) {
3634 + struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
3635 +
3636 + fsl_chan->qdma = fsl_qdma;
3637 + fsl_chan->queue = fsl_qdma->queue + i % fsl_qdma->n_queues;
3638 + fsl_chan->vchan.desc_free = fsl_qdma_free_desc;
3639 + INIT_LIST_HEAD(&fsl_chan->qcomp);
3640 + vchan_init(&fsl_chan->vchan, &fsl_qdma->dma_dev);
3641 + }
3642 + for (i = 0; i < fsl_qdma->n_queues; i++)
3643 + fsl_qdma_pre_request_enqueue_desc(fsl_qdma->queue + i);
3644 +
3645 + dma_cap_set(DMA_MEMCPY, fsl_qdma->dma_dev.cap_mask);
3646 + dma_cap_set(DMA_SG, fsl_qdma->dma_dev.cap_mask);
3647 +
3648 + fsl_qdma->dma_dev.dev = &pdev->dev;
3649 + fsl_qdma->dma_dev.device_alloc_chan_resources
3650 + = fsl_qdma_alloc_chan_resources;
3651 + fsl_qdma->dma_dev.device_free_chan_resources
3652 + = fsl_qdma_free_chan_resources;
3653 + fsl_qdma->dma_dev.device_tx_status = fsl_qdma_tx_status;
3654 + fsl_qdma->dma_dev.device_prep_dma_memcpy = fsl_qdma_prep_memcpy;
3655 + fsl_qdma->dma_dev.device_prep_dma_sg = fsl_qdma_prep_dma_sg;
3656 + fsl_qdma->dma_dev.device_issue_pending = fsl_qdma_issue_pending;
3657 +
3658 + dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));
3659 +
3660 + platform_set_drvdata(pdev, fsl_qdma);
3661 +
3662 + ret = dma_async_device_register(&fsl_qdma->dma_dev);
3663 + if (ret) {
3664 + dev_err(&pdev->dev, "Can't register Freescale qDMA engine.\n");
3665 + return ret;
3666 + }
3667 +
3668 + ret = fsl_qdma_reg_init(fsl_qdma);
3669 + if (ret) {
3670 + dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n");
3671 + return ret;
3672 + }
3673 +
3674 +
3675 + return 0;
3676 +}
3677 +
3678 +static int fsl_qdma_remove(struct platform_device *pdev)
3679 +{
3680 + struct device_node *np = pdev->dev.of_node;
3681 + struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev);
3682 + struct fsl_qdma_queue *queue_temp;
3683 + struct fsl_qdma_queue *status = fsl_qdma->status;
3684 + struct fsl_qdma_comp *comp_temp, *_comp_temp;
3685 + int i;
3686 +
3687 + of_dma_controller_free(np);
3688 + dma_async_device_unregister(&fsl_qdma->dma_dev);
3689 +
3690 + /* Free descriptor areas */
3691 + for (i = 0; i < fsl_qdma->n_queues; i++) {
3692 + queue_temp = fsl_qdma->queue + i;
3693 + list_for_each_entry_safe(comp_temp, _comp_temp,
3694 + &queue_temp->comp_used, list) {
3695 + dma_pool_free(queue_temp->comp_pool,
3696 + comp_temp->virt_addr,
3697 + comp_temp->bus_addr);
3698 + list_del(&comp_temp->list);
3699 + kfree(comp_temp);
3700 + }
3701 + list_for_each_entry_safe(comp_temp, _comp_temp,
3702 + &queue_temp->comp_free, list) {
3703 + dma_pool_free(queue_temp->comp_pool,
3704 + comp_temp->virt_addr,
3705 + comp_temp->bus_addr);
3706 + list_del(&comp_temp->list);
3707 + kfree(comp_temp);
3708 + }
3709 + dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_ccdf) *
3710 + queue_temp->n_cq, queue_temp->cq,
3711 + queue_temp->bus_addr);
3712 + dma_pool_destroy(queue_temp->comp_pool);
3713 + }
3714 +
3715 + dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_ccdf) *
3716 + status->n_cq, status->cq, status->bus_addr);
3717 + return 0;
3718 +}
3719 +
3720 +static const struct of_device_id fsl_qdma_dt_ids[] = {
3721 + { .compatible = "fsl,ls1021a-qdma", },
3722 + { /* sentinel */ }
3723 +};
3724 +MODULE_DEVICE_TABLE(of, fsl_qdma_dt_ids);
3725 +
3726 +static struct platform_driver fsl_qdma_driver = {
3727 + .driver = {
3728 + .name = "fsl-qdma",
3729 + .owner = THIS_MODULE,
3730 + .of_match_table = fsl_qdma_dt_ids,
3731 + },
3732 + .probe = fsl_qdma_probe,
3733 + .remove = fsl_qdma_remove,
3734 +};
3735 +
3736 +static int __init fsl_qdma_init(void)
3737 +{
3738 + return platform_driver_register(&fsl_qdma_driver);
3739 +}
3740 +subsys_initcall(fsl_qdma_init);
3741 +
3742 +static void __exit fsl_qdma_exit(void)
3743 +{
3744 + platform_driver_unregister(&fsl_qdma_driver);
3745 +}
3746 +module_exit(fsl_qdma_exit);
3747 +
3748 +MODULE_ALIAS("platform:fsl-qdma");
3749 +MODULE_DESCRIPTION("Freescale qDMA engine driver");
3750 +MODULE_LICENSE("GPL v2");
Staging tree of dedeckeh