target/linux/bcm27xx/patches-6.1/950-1208-media-rp1-cfe-Dual-purpose-video-nodes.patch

   1 From c54b8d2fc79c684deacc81a94f6baa1cb56c62be Mon Sep 17 00:00:00 2001
   2 From: Tomi Valkeinen <tomi.valkeinen@ideasonboard.com>
   3 Date: Wed, 27 Sep 2023 17:18:09 +0300
   4 Subject: [PATCH] media: rp1: cfe: Dual purpose video nodes
   5
   6 The RP1 CSI-2 DMA can capture both video and metadata just fine, but at
   7 the moment the video nodes are only set to support either video or
   8 metadata.
   9
  10 Make the changes to support both video and metadata. This mostly means
  11 tracking both video format and metadata format separately for each video
  12 node, and using vb2_queue_change_type() to change the vb2 queue type
  13 when needed.
  14
  15 Briefly, this means that the user can get/set both video and meta
  16 formats to a single video node. The vb2 queue buffer type will be
  17 changed when the user calls REQBUFS or CREATE_BUFS ioctls. This buffer
  18 type will be then used as the "mode" for the video node when the user
  19 starts the streaming, and based on that either the video or the meta
  20 format will be used.
  21
  22 A bunch of macros are added (node_supports_xxx()), which tell if a node
  23 can support a particular mode, whereas the existing macros
  24 (is_xxx_node()) will tell if the node is currently in a particular mode.
  25 Note that the latter will only work correctly between the start of the
  26 streaming and the end of the streaming, and thus should be only used in
  27 those code paths.
  28
  29 However, as the userspace (libcamera) does not support dual purpose
  30 video nodes, for the time being let's keep the second video node as
  31 V4L2_CAP_META_CAPTURE only to keep the userspace working.
  32
  33 Signed-off-by: Tomi Valkeinen <tomi.valkeinen@ideasonboard.com>
  34 ---
  35  .../media/platform/raspberrypi/rp1_cfe/cfe.c  | 271 ++++++++++++------
  36  1 file changed, 182 insertions(+), 89 deletions(-)
  37
  38 --- a/drivers/media/platform/raspberrypi/rp1_cfe/cfe.c
  39 +++ b/drivers/media/platform/raspberrypi/rp1_cfe/cfe.c
  40 @@ -116,7 +116,7 @@ const struct v4l2_mbus_framefmt cfe_defa
  41  enum node_ids {
  42         /* CSI2 HW output nodes first. */
  43         CSI2_CH0,
  44 -       CSI2_CH1_EMBEDDED,
  45 +       CSI2_CH1,
  46         CSI2_CH2,
  47         CSI2_CH3,
  48         /* FE only nodes from here on. */
  49 @@ -130,8 +130,7 @@ enum node_ids {
  50  struct node_description {
  51         unsigned int id;
  52         const char *name;
  53 -       enum v4l2_buf_type buf_type;
  54 -       unsigned int cap;
  55 +       unsigned int caps;
  56         unsigned int pad_flags;
  57         unsigned int link_pad;
  58  };
  59 @@ -140,58 +139,55 @@ struct node_description {
  60  static const struct node_description node_desc[NUM_NODES] = {
  61         [CSI2_CH0] = {
  62                 .name = "csi2_ch0",
  63 -               .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
  64 -               .cap = V4L2_CAP_VIDEO_CAPTURE,
  65 +               .caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_META_CAPTURE,
  66                 .pad_flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT,
  67                 .link_pad = CSI2_NUM_CHANNELS + 0
  68         },
  69 -       /* This node is assigned for the embedded data channel! */
  70 -       [CSI2_CH1_EMBEDDED] = {
  71 +       /*
  72 +        * TODO: This node should be named "csi2_ch1" and the caps should be set
  73 +        * to both video and meta capture. However, to keep compatibility with
  74 +        * the current libcamera, keep the name as "embedded" and support
  75 +        * only meta capture.
  76 +        */
  77 +       [CSI2_CH1] = {
  78                 .name = "embedded",
  79 -               .buf_type = V4L2_BUF_TYPE_META_CAPTURE,
  80 -               .cap = V4L2_CAP_META_CAPTURE,
  81 +               .caps = V4L2_CAP_META_CAPTURE,
  82                 .pad_flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT,
  83                 .link_pad = CSI2_NUM_CHANNELS + 1
  84         },
  85         [CSI2_CH2] = {
  86                 .name = "csi2_ch2",
  87 -               .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
  88 -               .cap = V4L2_CAP_META_CAPTURE,
  89 +               .caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_META_CAPTURE,
  90                 .pad_flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT,
  91                 .link_pad = CSI2_NUM_CHANNELS + 2
  92         },
  93         [CSI2_CH3] = {
  94                 .name = "csi2_ch3",
  95 -               .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
  96 -               .cap = V4L2_CAP_META_CAPTURE,
  97 +               .caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_META_CAPTURE,
  98                 .pad_flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT,
  99                 .link_pad = CSI2_NUM_CHANNELS + 3
 100         },
 101         [FE_OUT0] = {
 102                 .name = "fe_image0",
 103 -               .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
 104 -               .cap = V4L2_CAP_VIDEO_CAPTURE,
 105 +               .caps = V4L2_CAP_VIDEO_CAPTURE,
 106                 .pad_flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT,
 107                 .link_pad = FE_OUTPUT0_PAD
 108         },
 109         [FE_OUT1] = {
 110                 .name = "fe_image1",
 111 -               .buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
 112 -               .cap = V4L2_CAP_VIDEO_CAPTURE,
 113 +               .caps = V4L2_CAP_VIDEO_CAPTURE,
 114                 .pad_flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT,
 115                 .link_pad = FE_OUTPUT1_PAD
 116         },
 117         [FE_STATS] = {
 118                 .name = "fe_stats",
 119 -               .buf_type = V4L2_BUF_TYPE_META_CAPTURE,
 120 -               .cap = V4L2_CAP_META_CAPTURE,
 121 +               .caps = V4L2_CAP_META_CAPTURE,
 122                 .pad_flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT,
 123                 .link_pad = FE_STATS_PAD
 124         },
 125         [FE_CONFIG] = {
 126                 .name = "fe_config",
 127 -               .buf_type = V4L2_BUF_TYPE_META_OUTPUT,
 128 -               .cap = V4L2_CAP_META_OUTPUT,
 129 +               .caps = V4L2_CAP_META_OUTPUT,
 130                 .pad_flags = MEDIA_PAD_FL_SOURCE | MEDIA_PAD_FL_MUST_CONNECT,
 131                 .link_pad = FE_CONFIG_PAD
 132         },
 133 @@ -200,17 +196,29 @@ static const struct node_description nod
 134  #define is_fe_node(node) (((node)->id) >= FE_OUT0)
 135  #define is_csi2_node(node) (!is_fe_node(node))
 136
 137 +#define node_supports_image_output(node) \
 138 +       (!!(node_desc[(node)->id].caps & V4L2_CAP_VIDEO_CAPTURE))
 139 +#define node_supports_meta_output(node) \
 140 +       (!!(node_desc[(node)->id].caps & V4L2_CAP_META_CAPTURE))
 141 +#define node_supports_image_input(node) \
 142 +       (!!(node_desc[(node)->id].caps & V4L2_CAP_VIDEO_OUTPUT))
 143 +#define node_supports_meta_input(node) \
 144 +       (!!(node_desc[(node)->id].caps & V4L2_CAP_META_OUTPUT))
 145 +#define node_supports_image(node) \
 146 +       (node_supports_image_output(node) || node_supports_image_input(node))
 147 +#define node_supports_meta(node) \
 148 +       (node_supports_meta_output(node) || node_supports_meta_input(node))
 149 +
 150  #define is_image_output_node(node) \
 151 -       (node_desc[(node)->id].buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
 152 +       ((node)->buffer_queue.type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
 153  #define is_image_input_node(node) \
 154 -       (node_desc[(node)->id].buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
 155 +       ((node)->buffer_queue.type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
 156  #define is_image_node(node) \
 157         (is_image_output_node(node) || is_image_input_node(node))
 158 -
 159  #define is_meta_output_node(node) \
 160 -       (node_desc[(node)->id].buf_type == V4L2_BUF_TYPE_META_CAPTURE)
 161 +       ((node)->buffer_queue.type == V4L2_BUF_TYPE_META_CAPTURE)
 162  #define is_meta_input_node(node) \
 163 -       (node_desc[(node)->id].buf_type == V4L2_BUF_TYPE_META_OUTPUT)
 164 +       ((node)->buffer_queue.type == V4L2_BUF_TYPE_META_OUTPUT)
 165  #define is_meta_node(node) \
 166         (is_meta_output_node(node) || is_meta_input_node(node))
 167
 168 @@ -250,7 +258,9 @@ struct cfe_node {
 169         /* Pointer pointing to next v4l2_buffer */
 170         struct cfe_buffer *next_frm;
 171         /* Used to store current pixel format */
 172 -       struct v4l2_format fmt;
 173 +       struct v4l2_format vid_fmt;
 174 +       /* Used to store current meta format */
 175 +       struct v4l2_format meta_fmt;
 176         /* Buffer queue used in video-buf */
 177         struct vb2_queue buffer_queue;
 178         /* Queue of filled frames */
 179 @@ -433,20 +443,21 @@ static int format_show(struct seq_file *
 180                 seq_printf(s, "\nNode %u (%s) state: 0x%lx\n", i,
 181                            node_desc[i].name, state);
 182
 183 -               if (is_image_node(node))
 184 +               if (node_supports_image(node))
 185                         seq_printf(s, "format: " V4L2_FOURCC_CONV " 0x%x\n"
 186                                       "resolution: %ux%u\nbpl: %u\nsize: %u\n",
 187 -                                  V4L2_FOURCC_CONV_ARGS(node->fmt.fmt.pix.pixelformat),
 188 -                                  node->fmt.fmt.pix.pixelformat,
 189 -                                  node->fmt.fmt.pix.width,
 190 -                                  node->fmt.fmt.pix.height,
 191 -                                  node->fmt.fmt.pix.bytesperline,
 192 -                                  node->fmt.fmt.pix.sizeimage);
 193 -               else
 194 +                                  V4L2_FOURCC_CONV_ARGS(node->vid_fmt.fmt.pix.pixelformat),
 195 +                                  node->vid_fmt.fmt.pix.pixelformat,
 196 +                                  node->vid_fmt.fmt.pix.width,
 197 +                                  node->vid_fmt.fmt.pix.height,
 198 +                                  node->vid_fmt.fmt.pix.bytesperline,
 199 +                                  node->vid_fmt.fmt.pix.sizeimage);
 200 +
 201 +               if (node_supports_meta(node))
 202                         seq_printf(s, "format: " V4L2_FOURCC_CONV " 0x%x\nsize: %u\n",
 203 -                                  V4L2_FOURCC_CONV_ARGS(node->fmt.fmt.meta.dataformat),
 204 -                                  node->fmt.fmt.meta.dataformat,
 205 -                                  node->fmt.fmt.meta.buffersize);
 206 +                                  V4L2_FOURCC_CONV_ARGS(node->meta_fmt.fmt.meta.dataformat),
 207 +                                  node->meta_fmt.fmt.meta.dataformat,
 208 +                                  node->meta_fmt.fmt.meta.buffersize);
 209         }
 210
 211         return 0;
 212 @@ -571,11 +582,11 @@ static void cfe_schedule_next_csi2_job(s
 213                                 node_desc[node->id].name, &buf->vb.vb2_buf);
 214
 215                 if (is_meta_node(node)) {
 216 -                       size = node->fmt.fmt.meta.buffersize;
 217 +                       size = node->meta_fmt.fmt.meta.buffersize;
 218                         stride = 0;
 219                 } else {
 220 -                       size = node->fmt.fmt.pix.sizeimage;
 221 -                       stride = node->fmt.fmt.pix.bytesperline;
 222 +                       size = node->vid_fmt.fmt.pix.sizeimage;
 223 +                       stride = node->vid_fmt.fmt.pix.bytesperline;
 224                 }
 225
 226                 addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
 227 @@ -867,10 +878,10 @@ static void cfe_start_channel(struct cfe
 228                         width = source_fmt->width;
 229                         height = source_fmt->height;
 230
 231 -                       if (node->fmt.fmt.pix.pixelformat ==
 232 +                       if (node->vid_fmt.fmt.pix.pixelformat ==
 233                                         fmt->remap[CFE_REMAP_16BIT])
 234                                 mode = CSI2_MODE_REMAP;
 235 -                       else if (node->fmt.fmt.pix.pixelformat ==
 236 +                       else if (node->vid_fmt.fmt.pix.pixelformat ==
 237                                         fmt->remap[CFE_REMAP_COMPRESSED]) {
 238                                 mode = CSI2_MODE_COMPRESSED;
 239                                 csi2_set_compression(&cfe->csi2, node->id,
 240 @@ -884,7 +895,7 @@ static void cfe_start_channel(struct cfe
 241                                    /* Auto arm */
 242                                    false,
 243                                    /* Pack bytes */
 244 -                                  node->id == CSI2_CH1_EMBEDDED ? true : false,
 245 +                                  is_meta_node(node) ? true : false,
 246                                    width, height);
 247         }
 248
 249 @@ -947,10 +958,11 @@ static int cfe_queue_setup(struct vb2_qu
 250  {
 251         struct cfe_node *node = vb2_get_drv_priv(vq);
 252         struct cfe_device *cfe = node->cfe;
 253 -       unsigned int size = is_image_node(node) ? node->fmt.fmt.pix.sizeimage :
 254 -                                                 node->fmt.fmt.meta.buffersize;
 255 +       unsigned int size = is_image_node(node) ? node->vid_fmt.fmt.pix.sizeimage :
 256 +                                                 node->meta_fmt.fmt.meta.buffersize;
 257
 258 -       cfe_dbg("%s: [%s]\n", __func__, node_desc[node->id].name);
 259 +       cfe_dbg("%s: [%s] type:%u\n", __func__, node_desc[node->id].name,
 260 +               node->buffer_queue.type);
 261
 262         if (vq->num_buffers + *nbuffers < 3)
 263                 *nbuffers = 3 - vq->num_buffers;
 264 @@ -979,8 +991,8 @@ static int cfe_buffer_prepare(struct vb2
 265         cfe_dbg_verbose("%s: [%s] buffer:%p\n", __func__,
 266                         node_desc[node->id].name, vb);
 267
 268 -       size = is_image_node(node) ? node->fmt.fmt.pix.sizeimage :
 269 -                                    node->fmt.fmt.meta.buffersize;
 270 +       size = is_image_node(node) ? node->vid_fmt.fmt.pix.sizeimage :
 271 +                                    node->meta_fmt.fmt.meta.buffersize;
 272         if (vb2_plane_size(vb, 0) < size) {
 273                 cfe_err("data will not fit into plane (%lu < %lu)\n",
 274                         vb2_plane_size(vb, 0), size);
 275 @@ -995,8 +1007,8 @@ static int cfe_buffer_prepare(struct vb2
 276
 277                 memcpy(&b->config, addr, sizeof(struct pisp_fe_config));
 278                 return pisp_fe_validate_config(&cfe->fe, &b->config,
 279 -                                              &cfe->node[FE_OUT0].fmt,
 280 -                                              &cfe->node[FE_OUT1].fmt);
 281 +                                              &cfe->node[FE_OUT0].vid_fmt,
 282 +                                              &cfe->node[FE_OUT1].vid_fmt);
 283         }
 284
 285         return 0;
 286 @@ -1256,7 +1268,7 @@ static int cfe_enum_fmt_vid_cap(struct f
 287         struct cfe_device *cfe = node->cfe;
 288         unsigned int i, j;
 289
 290 -       if (!is_image_output_node(node))
 291 +       if (!node_supports_image_output(node))
 292                 return -EINVAL;
 293
 294         cfe_dbg("%s: [%s]\n", __func__, node_desc[node->id].name);
 295 @@ -1292,10 +1304,10 @@ static int cfe_g_fmt(struct file *file,
 296
 297         cfe_dbg("%s: [%s]\n", __func__, node_desc[node->id].name);
 298
 299 -       if (f->type != node->buffer_queue.type)
 300 +       if (!node_supports_image(node))
 301                 return -EINVAL;
 302
 303 -       *f = node->fmt;
 304 +       *f = node->vid_fmt;
 305
 306         return 0;
 307  }
 308 @@ -1310,7 +1322,7 @@ static int try_fmt_vid_cap(struct cfe_no
 309                 f->fmt.pix.width, f->fmt.pix.height,
 310                 V4L2_FOURCC_CONV_ARGS(f->fmt.pix.pixelformat));
 311
 312 -       if (!is_image_output_node(node))
 313 +       if (!node_supports_image_output(node))
 314                 return -EINVAL;
 315
 316         /*
 317 @@ -1351,11 +1363,11 @@ static int cfe_s_fmt_vid_cap(struct file
 318         if (ret)
 319                 return ret;
 320
 321 -       node->fmt = *f;
 322 +       node->vid_fmt = *f;
 323
 324         cfe_dbg("%s: Set %ux%u, V4L2 pix " V4L2_FOURCC_CONV "\n", __func__,
 325 -               node->fmt.fmt.pix.width, node->fmt.fmt.pix.height,
 326 -               V4L2_FOURCC_CONV_ARGS(node->fmt.fmt.pix.pixelformat));
 327 +               node->vid_fmt.fmt.pix.width, node->vid_fmt.fmt.pix.height,
 328 +               V4L2_FOURCC_CONV_ARGS(node->vid_fmt.fmt.pix.pixelformat));
 329
 330         return 0;
 331  }
 332 @@ -1379,11 +1391,11 @@ static int cfe_enum_fmt_meta(struct file
 333
 334         cfe_dbg("%s: [%s]\n", __func__, node_desc[node->id].name);
 335
 336 -       if (!is_meta_node(node) || f->index != 0)
 337 +       if (!node_supports_meta(node) || f->index != 0)
 338                 return -EINVAL;
 339
 340         switch (node->id) {
 341 -       case CSI2_CH1_EMBEDDED:
 342 +       case CSI2_CH0...CSI2_CH3:
 343                 f->pixelformat = V4L2_META_FMT_SENSOR_DATA;
 344                 return 0;
 345         case FE_STATS:
 346 @@ -1399,8 +1411,11 @@ static int cfe_enum_fmt_meta(struct file
 347
 348  static int try_fmt_meta(struct cfe_node *node, struct v4l2_format *f)
 349  {
 350 +       if (!node_supports_meta(node))
 351 +               return -EINVAL;
 352 +
 353         switch (node->id) {
 354 -       case CSI2_CH1_EMBEDDED:
 355 +       case CSI2_CH0...CSI2_CH3:
 356                 f->fmt.meta.dataformat = V4L2_META_FMT_SENSOR_DATA;
 357                 if (!f->fmt.meta.buffersize)
 358                         f->fmt.meta.buffersize = DEFAULT_EMBEDDED_SIZE;
 359 @@ -1422,6 +1437,21 @@ static int try_fmt_meta(struct cfe_node
 360         return -EINVAL;
 361  }
 362
 363 +static int cfe_g_fmt_meta(struct file *file, void *priv, struct v4l2_format *f)
 364 +{
 365 +       struct cfe_node *node = video_drvdata(file);
 366 +       struct cfe_device *cfe = node->cfe;
 367 +
 368 +       cfe_dbg("%s: [%s]\n", __func__, node_desc[node->id].name);
 369 +
 370 +       if (!node_supports_meta(node))
 371 +               return -EINVAL;
 372 +
 373 +       *f = node->meta_fmt;
 374 +
 375 +       return 0;
 376 +}
 377 +
 378  static int cfe_s_fmt_meta(struct file *file, void *priv, struct v4l2_format *f)
 379  {
 380         struct cfe_node *node = video_drvdata(file);
 381 @@ -1434,17 +1464,17 @@ static int cfe_s_fmt_meta(struct file *f
 382         if (vb2_is_busy(q))
 383                 return -EBUSY;
 384
 385 -       if (f->type != node->buffer_queue.type)
 386 +       if (!node_supports_meta(node))
 387                 return -EINVAL;
 388
 389         ret = try_fmt_meta(node, f);
 390         if (ret)
 391                 return ret;
 392
 393 -       node->fmt = *f;
 394 +       node->meta_fmt = *f;
 395
 396         cfe_dbg("%s: Set " V4L2_FOURCC_CONV "\n", __func__,
 397 -               V4L2_FOURCC_CONV_ARGS(node->fmt.fmt.meta.dataformat));
 398 +               V4L2_FOURCC_CONV_ARGS(node->meta_fmt.fmt.meta.dataformat));
 399
 400         return 0;
 401  }
 402 @@ -1491,6 +1521,52 @@ static int cfe_enum_framesizes(struct fi
 403         return 0;
 404  }
 405
 406 +static int cfe_vb2_ioctl_reqbufs(struct file *file, void *priv,
 407 +                                struct v4l2_requestbuffers *p)
 408 +{
 409 +       struct video_device *vdev = video_devdata(file);
 410 +       struct cfe_node *node = video_get_drvdata(vdev);
 411 +       struct cfe_device *cfe = node->cfe;
 412 +       int ret;
 413 +
 414 +       cfe_dbg("%s: [%s] type:%u\n", __func__, node_desc[node->id].name,
 415 +               p->type);
 416 +
 417 +       if (p->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
 418 +           p->type != V4L2_BUF_TYPE_META_CAPTURE &&
 419 +           p->type != V4L2_BUF_TYPE_META_OUTPUT)
 420 +               return -EINVAL;
 421 +
 422 +       ret = vb2_queue_change_type(vdev->queue, p->type);
 423 +       if (ret)
 424 +               return ret;
 425 +
 426 +       return vb2_ioctl_reqbufs(file, priv, p);
 427 +}
 428 +
 429 +static int cfe_vb2_ioctl_create_bufs(struct file *file, void *priv,
 430 +                                    struct v4l2_create_buffers *p)
 431 +{
 432 +       struct video_device *vdev = video_devdata(file);
 433 +       struct cfe_node *node = video_get_drvdata(vdev);
 434 +       struct cfe_device *cfe = node->cfe;
 435 +       int ret;
 436 +
 437 +       cfe_dbg("%s: [%s] type:%u\n", __func__, node_desc[node->id].name,
 438 +               p->format.type);
 439 +
 440 +       if (p->format.type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
 441 +           p->format.type != V4L2_BUF_TYPE_META_CAPTURE &&
 442 +           p->format.type != V4L2_BUF_TYPE_META_OUTPUT)
 443 +               return -EINVAL;
 444 +
 445 +       ret = vb2_queue_change_type(vdev->queue, p->format.type);
 446 +       if (ret)
 447 +               return ret;
 448 +
 449 +       return vb2_ioctl_create_bufs(file, priv, p);
 450 +}
 451 +
 452  static int cfe_subscribe_event(struct v4l2_fh *fh,
 453                                const struct v4l2_event_subscription *sub)
 454  {
 455 @@ -1498,12 +1574,13 @@ static int cfe_subscribe_event(struct v4
 456
 457         switch (sub->type) {
 458         case V4L2_EVENT_FRAME_SYNC:
 459 -               if (!is_image_output_node(node))
 460 +               if (!node_supports_image_output(node))
 461                         break;
 462
 463                 return v4l2_event_subscribe(fh, sub, 2, NULL);
 464         case V4L2_EVENT_SOURCE_CHANGE:
 465 -               if (is_meta_input_node(node))
 466 +               if (!node_supports_image_output(node) &&
 467 +                   !node_supports_meta_output(node))
 468                         break;
 469
 470                 return v4l2_event_subscribe(fh, sub, 4, NULL);
 471 @@ -1520,19 +1597,19 @@ static const struct v4l2_ioctl_ops cfe_i
 472         .vidioc_try_fmt_vid_cap = cfe_try_fmt_vid_cap,
 473
 474         .vidioc_enum_fmt_meta_cap = cfe_enum_fmt_meta,
 475 -       .vidioc_g_fmt_meta_cap = cfe_g_fmt,
 476 +       .vidioc_g_fmt_meta_cap = cfe_g_fmt_meta,
 477         .vidioc_s_fmt_meta_cap = cfe_s_fmt_meta,
 478         .vidioc_try_fmt_meta_cap = cfe_try_fmt_meta,
 479
 480         .vidioc_enum_fmt_meta_out = cfe_enum_fmt_meta,
 481 -       .vidioc_g_fmt_meta_out = cfe_g_fmt,
 482 +       .vidioc_g_fmt_meta_out = cfe_g_fmt_meta,
 483         .vidioc_s_fmt_meta_out = cfe_s_fmt_meta,
 484         .vidioc_try_fmt_meta_out = cfe_try_fmt_meta,
 485
 486         .vidioc_enum_framesizes = cfe_enum_framesizes,
 487
 488 -       .vidioc_reqbufs = vb2_ioctl_reqbufs,
 489 -       .vidioc_create_bufs = vb2_ioctl_create_bufs,
 490 +       .vidioc_reqbufs = cfe_vb2_ioctl_reqbufs,
 491 +       .vidioc_create_bufs = cfe_vb2_ioctl_create_bufs,
 492         .vidioc_prepare_buf = vb2_ioctl_prepare_buf,
 493         .vidioc_querybuf = vb2_ioctl_querybuf,
 494         .vidioc_qbuf = vb2_ioctl_qbuf,
 495 @@ -1610,7 +1687,7 @@ static int cfe_video_link_validate(struc
 496         }
 497
 498         if (is_image_output_node(node)) {
 499 -               struct v4l2_pix_format *pix_fmt = &node->fmt.fmt.pix;
 500 +               struct v4l2_pix_format *pix_fmt = &node->vid_fmt.fmt.pix;
 501                 const struct cfe_fmt *fmt = NULL;
 502                 unsigned int i;
 503
 504 @@ -1636,8 +1713,8 @@ static int cfe_video_link_validate(struc
 505                         ret = -EINVAL;
 506                         goto out;
 507                 }
 508 -       } else if (node->id == CSI2_CH1_EMBEDDED) {
 509 -               struct v4l2_meta_format *meta_fmt = &node->fmt.fmt.meta;
 510 +       } else if (is_csi2_node(node) && is_meta_output_node(node)) {
 511 +               struct v4l2_meta_format *meta_fmt = &node->meta_fmt.fmt.meta;
 512
 513                 if (source_fmt->width * source_fmt->height !=
 514                                                         meta_fmt->buffersize ||
 515 @@ -1698,15 +1775,17 @@ static int cfe_video_link_notify(struct
 516
 517         if (link->source->entity != csi2)
 518                 return 0;
 519 -       if (link->sink->index != 0)
 520 +       if (link->sink->entity != fe)
 521                 return 0;
 522 -       if (link->source->index == node_desc[CSI2_CH1_EMBEDDED].link_pad)
 523 +       if (link->sink->index != 0)
 524                 return 0;
 525
 526         cfe->fe_csi2_channel = -1;
 527 -       if (link->sink->entity == fe && (link->flags & MEDIA_LNK_FL_ENABLED)) {
 528 +       if (link->flags & MEDIA_LNK_FL_ENABLED) {
 529                 if (link->source->index == node_desc[CSI2_CH0].link_pad)
 530                         cfe->fe_csi2_channel = CSI2_CH0;
 531 +               else if (link->source->index == node_desc[CSI2_CH1].link_pad)
 532 +                       cfe->fe_csi2_channel = CSI2_CH1;
 533                 else if (link->source->index == node_desc[CSI2_CH2].link_pad)
 534                         cfe->fe_csi2_channel = CSI2_CH2;
 535                 else if (link->source->index == node_desc[CSI2_CH3].link_pad)
 536 @@ -1763,30 +1842,42 @@ static int cfe_register_node(struct cfe_
 537         node->cfe = cfe;
 538         node->id = id;
 539
 540 -       if (is_image_node(node)) {
 541 +       if (node_supports_image(node)) {
 542 +               if (node_supports_image_output(node))
 543 +                       node->vid_fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 544 +               else
 545 +                       node->vid_fmt.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
 546 +
 547                 fmt = find_format_by_code(cfe_default_format.code);
 548                 if (!fmt) {
 549                         cfe_err("Failed to find format code\n");
 550                         return -EINVAL;
 551                 }
 552
 553 -               node->fmt.fmt.pix.pixelformat = fmt->fourcc;
 554 -               v4l2_fill_pix_format(&node->fmt.fmt.pix, &cfe_default_format);
 555 +               node->vid_fmt.fmt.pix.pixelformat = fmt->fourcc;
 556 +               v4l2_fill_pix_format(&node->vid_fmt.fmt.pix, &cfe_default_format);
 557
 558 -               ret = try_fmt_vid_cap(node, &node->fmt);
 559 +               ret = try_fmt_vid_cap(node, &node->vid_fmt);
 560                 if (ret)
 561                         return ret;
 562 -       } else {
 563 -               ret = try_fmt_meta(node, &node->fmt);
 564 +       }
 565 +
 566 +       if (node_supports_meta(node)) {
 567 +               if (node_supports_meta_output(node))
 568 +                       node->meta_fmt.type = V4L2_BUF_TYPE_META_CAPTURE;
 569 +               else
 570 +                       node->meta_fmt.type = V4L2_BUF_TYPE_META_OUTPUT;
 571 +
 572 +               ret = try_fmt_meta(node, &node->meta_fmt);
 573                 if (ret)
 574                         return ret;
 575         }
 576 -       node->fmt.type = node_desc[id].buf_type;
 577
 578         mutex_init(&node->lock);
 579
 580         q = &node->buffer_queue;
 581 -       q->type = node_desc[id].buf_type;
 582 +       q->type = node_supports_image(node) ? node->vid_fmt.type :
 583 +                                             node->meta_fmt.type;
 584         q->io_modes = VB2_MMAP | VB2_DMABUF;
 585         q->drv_priv = node;
 586         q->ops = &cfe_video_qops;
 587 @@ -1812,11 +1903,13 @@ static int cfe_register_node(struct cfe_
 588         vdev->ioctl_ops = &cfe_ioctl_ops;
 589         vdev->entity.ops = &cfe_media_entity_ops;
 590         vdev->v4l2_dev = &cfe->v4l2_dev;
 591 -       vdev->vfl_dir = (is_image_output_node(node) || is_meta_output_node(node))
 592 -                               ? VFL_DIR_RX : VFL_DIR_TX;
 593 +       vdev->vfl_dir = (node_supports_image_output(node) ||
 594 +                        node_supports_meta_output(node)) ?
 595 +                               VFL_DIR_RX :
 596 +                               VFL_DIR_TX;
 597         vdev->queue = q;
 598         vdev->lock = &node->lock;
 599 -       vdev->device_caps = node_desc[id].cap;
 600 +       vdev->device_caps = node_desc[id].caps;
 601         vdev->device_caps |= V4L2_CAP_STREAMING | V4L2_CAP_IO_MC;
 602
 603         /* Define the device names */
 604 @@ -1829,7 +1922,7 @@ static int cfe_register_node(struct cfe_
 605         node->pad.flags = node_desc[id].pad_flags;
 606         media_entity_pads_init(&vdev->entity, 1, &node->pad);
 607
 608 -       if (is_meta_node(node)) {
 609 +       if (!node_supports_image(node)) {
 610                 v4l2_disable_ioctl(&node->video_dev,
 611                                    VIDIOC_ENUM_FRAMEINTERVALS);
 612                 v4l2_disable_ioctl(&node->video_dev,
 613 @@ -1907,7 +2000,7 @@ static int cfe_link_node_pads(struct cfe
 614                 if (ret)
 615                         return ret;
 616
 617 -               if (node->id != CSI2_CH1_EMBEDDED) {
 618 +               if (node_supports_image(node)) {
 619                         /* CSI2 channel # -> FE Input */
 620                         ret = media_create_pad_link(&cfe->csi2.sd.entity,
 621                                                     node_desc[i].link_pad,