target/linux/bcm27xx/patches-6.1/950-0917-drivers-iommu-Add-BCM2712-IOMMU.patch

   1 From 5fd6ee7fd084838e09d4e463ae53cd9aaa7fce70 Mon Sep 17 00:00:00 2001
   2 From: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>
   3 Date: Thu, 11 May 2023 16:37:34 +0100
   4 Subject: [PATCH] drivers: iommu: Add BCM2712 IOMMU
   5
   6 Add a driver for BCM2712 IOMMUs.
   7 There is a small driver for the Shared IOMMU TLB Cache.
   8 Each IOMMU instance is a separate device.
   9
  10 IOMMUs are set up with a "pass-through" range covering
  11 the lowest 40BGytes (which should cover all of SDRAM)
  12 for the benefit of non-IOMMU-aware devices that share
  13 a physical IOMMU; and translation for addresses in the
  14 range 40GB to 42GB.
  15
  16 An optional parameter adds a DMA offset (which otherwise
  17 would be lost?) to virtual addresses for DMA masters on a
  18 bus such as PCIe.
  19
  20 Signed-off-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>
  21 ---
  22  drivers/iommu/Kconfig               |   7 +
  23  drivers/iommu/Makefile              |   1 +
  24  drivers/iommu/bcm2712-iommu-cache.c |  77 ++++
  25  drivers/iommu/bcm2712-iommu.c       | 672 ++++++++++++++++++++++++++++
  26  drivers/iommu/bcm2712-iommu.h       |  45 ++
  27  5 files changed, 802 insertions(+)
  28  create mode 100644 drivers/iommu/bcm2712-iommu-cache.c
  29  create mode 100644 drivers/iommu/bcm2712-iommu.c
  30  create mode 100644 drivers/iommu/bcm2712-iommu.h
  31
  32 --- a/drivers/iommu/Kconfig
  33 +++ b/drivers/iommu/Kconfig
  34 @@ -506,4 +506,11 @@ config SPRD_IOMMU
  35
  36           Say Y here if you want to use the multimedia devices listed above.
  37
  38 +config BCM2712_IOMMU
  39 +       tristate "BCM2712 IOMMU driver"
  40 +       depends on ARM64 && ARCH_BCM
  41 +       select IOMMU_API
  42 +       help
  43 +        IOMMU driver for BCM2712
  44 +
  45  endif # IOMMU_SUPPORT
  46 --- a/drivers/iommu/Makefile
  47 +++ b/drivers/iommu/Makefile
  48 @@ -31,3 +31,4 @@ obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iom
  49  obj-$(CONFIG_IOMMU_SVA) += iommu-sva-lib.o io-pgfault.o
  50  obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
  51  obj-$(CONFIG_APPLE_DART) += apple-dart.o
  52 +obj-$(CONFIG_BCM2712_IOMMU) += bcm2712-iommu.o bcm2712-iommu-cache.o
  53 --- /dev/null
  54 +++ b/drivers/iommu/bcm2712-iommu-cache.c
  55 @@ -0,0 +1,77 @@
  56 +// SPDX-License-Identifier: GPL-2.0-only
  57 +/*
  58 + * IOMMU driver for BCM2712
  59 + *
  60 + * Copyright (c) 2023 Raspberry Pi Ltd.
  61 + */
  62 +
  63 +#include "bcm2712-iommu.h"
  64 +
  65 +#include <linux/err.h>
  66 +#include <linux/of_platform.h>
  67 +#include <linux/platform_device.h>
  68 +#include <linux/spinlock.h>
  69 +
  70 +#define MMUC_CONTROL_ENABLE   1
  71 +#define MMUC_CONTROL_FLUSH    2
  72 +#define MMUC_CONTROL_FLUSHING 4
  73 +
  74 +void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache)
  75 +{
  76 +       unsigned long flags;
  77 +       int i;
  78 +
  79 +       spin_lock_irqsave(&cache->hw_lock, flags);
  80 +       if (cache->reg_base) {
  81 +               /* Enable and flush the TLB cache */
  82 +               writel(MMUC_CONTROL_ENABLE | MMUC_CONTROL_FLUSH,
  83 +                      cache->reg_base);
  84 +
  85 +               /* Wait for flush to complete: it should be very quick */
  86 +               for (i = 0; i < 1024; i++) {
  87 +                       if (!(MMUC_CONTROL_FLUSHING & readl(cache->reg_base)))
  88 +                               break;
  89 +                       cpu_relax();
  90 +               }
  91 +       }
  92 +       spin_unlock_irqrestore(&cache->hw_lock, flags);
  93 +}
  94 +
  95 +static int bcm2712_iommu_cache_probe(struct platform_device *pdev)
  96 +{
  97 +       struct bcm2712_iommu_cache *cache;
  98 +
  99 +       dev_info(&pdev->dev, __func__);
 100 +       cache = devm_kzalloc(&pdev->dev, sizeof(*cache), GFP_KERNEL);
 101 +       if (!cache)
 102 +               return -ENOMEM;
 103 +
 104 +       cache->dev = &pdev->dev;
 105 +       platform_set_drvdata(pdev, cache);
 106 +       spin_lock_init(&cache->hw_lock);
 107 +
 108 +       /* Get IOMMUC registers; we only use the first register (IOMMUC_CTRL) */
 109 +       cache->reg_base = devm_platform_ioremap_resource(pdev, 0);
 110 +       if (IS_ERR(cache->reg_base)) {
 111 +               dev_err(&pdev->dev, "Failed to get IOMMU Cache registers address\n");
 112 +               cache->reg_base = NULL;
 113 +       }
 114 +       return 0;
 115 +}
 116 +
 117 +static const struct of_device_id bcm2712_iommu_cache_of_match[] = {
 118 +       {
 119 +               . compatible = "brcm,bcm2712-iommuc"
 120 +       },
 121 +       { /* sentinel */ },
 122 +};
 123 +
 124 +static struct platform_driver bcm2712_iommu_cache_driver = {
 125 +       .probe = bcm2712_iommu_cache_probe,
 126 +       .driver = {
 127 +               .name = "bcm2712-iommu-cache",
 128 +               .of_match_table = bcm2712_iommu_cache_of_match
 129 +       },
 130 +};
 131 +
 132 +builtin_platform_driver(bcm2712_iommu_cache_driver);
 133 --- /dev/null
 134 +++ b/drivers/iommu/bcm2712-iommu.c
 135 @@ -0,0 +1,672 @@
 136 +// SPDX-License-Identifier: GPL-2.0-only
 137 +/*
 138 + * IOMMU driver for BCM2712
 139 + *
 140 + * Copyright (c) 2023 Raspberry Pi Ltd.
 141 + */
 142 +
 143 +#include "bcm2712-iommu.h"
 144 +
 145 +#include <linux/dma-mapping.h>
 146 +#include <linux/err.h>
 147 +#include <linux/iommu.h>
 148 +#include <linux/of_platform.h>
 149 +#include <linux/platform_device.h>
 150 +#include <linux/spinlock.h>
 151 +
 152 +#define MMU_WR(off, val)   writel(val, mmu->reg_base + (off))
 153 +#define MMU_RD(off)        readl(mmu->reg_base + (off))
 154 +
 155 +#define domain_to_mmu(d) (container_of(d, struct bcm2712_iommu_domain, base)->mmu)
 156 +
 157 +#define MMMU_CTRL_OFFSET                       0x00
 158 +#define MMMU_CTRL_CAP_EXCEEDED                 BIT(27)
 159 +#define MMMU_CTRL_CAP_EXCEEDED_ABORT_EN        BIT(26)
 160 +#define MMMU_CTRL_CAP_EXCEEDED_INT_EN          BIT(25)
 161 +#define MMMU_CTRL_CAP_EXCEEDED_EXCEPTION_EN    BIT(24)
 162 +#define MMMU_CTRL_PT_INVALID                   BIT(20)
 163 +#define MMMU_CTRL_PT_INVALID_ABORT_EN          BIT(19)
 164 +#define MMMU_CTRL_PT_INVALID_EXCEPTION_EN      BIT(18)
 165 +#define MMMU_CTRL_PT_INVALID_EN                BIT(17)
 166 +#define MMMU_CTRL_WRITE_VIOLATION              BIT(12)
 167 +#define MMMU_CTRL_WRITE_VIOLATION_ABORT_EN     BIT(11)
 168 +#define MMMU_CTRL_WRITE_VIOLATION_INT_EN       BIT(10)
 169 +#define MMMU_CTRL_WRITE_VIOLATION_EXCEPTION_EN BIT(9)
 170 +#define MMMU_CTRL_BYPASS                       BIT(8)
 171 +#define MMMU_CTRL_TLB_CLEARING                 BIT(7)
 172 +#define MMMU_CTRL_STATS_CLEAR                  BIT(3)
 173 +#define MMMU_CTRL_TLB_CLEAR                    BIT(2)
 174 +#define MMMU_CTRL_STATS_ENABLE                 BIT(1)
 175 +#define MMMU_CTRL_ENABLE                       BIT(0)
 176 +
 177 +#define MMMU_PT_PA_BASE_OFFSET                 0x04
 178 +
 179 +#define MMMU_HIT_OFFSET                        0x08
 180 +#define MMMU_MISS_OFFSET                       0x0C
 181 +#define MMMU_STALL_OFFSET                      0x10
 182 +
 183 +#define MMMU_ADDR_CAP_OFFSET                   0x14
 184 +#define MMMU_ADDR_CAP_ENABLE                   BIT(31)
 185 +#define ADDR_CAP_SHIFT 28 /* ADDR_CAP is defined to be in 256 MByte units */
 186 +
 187 +#define MMMU_SHOOT_DOWN_OFFSET                 0x18
 188 +#define MMMU_SHOOT_DOWN_SHOOTING               BIT(31)
 189 +#define MMMU_SHOOT_DOWN_SHOOT                  BIT(30)
 190 +
 191 +#define MMMU_BYPASS_START_OFFSET               0x1C
 192 +#define MMMU_BYPASS_START_ENABLE               BIT(31)
 193 +#define MMMU_BYPASS_START_INVERT               BIT(30)
 194 +
 195 +#define MMMU_BYPASS_END_OFFSET                 0x20
 196 +#define MMMU_BYPASS_END_ENABLE                 BIT(31)
 197 +
 198 +#define MMMU_MISC_OFFSET                       0x24
 199 +#define MMMU_MISC_SINGLE_TABLE                 BIT(31)
 200 +
 201 +#define MMMU_ILLEGAL_ADR_OFFSET                0x30
 202 +#define MMMU_ILLEGAL_ADR_ENABLE                BIT(31)
 203 +
 204 +#define MMMU_DEBUG_INFO_OFFSET                 0x38
 205 +#define MMMU_DEBUG_INFO_VERSION_MASK           0x0000000Fu
 206 +#define MMMU_DEBUG_INFO_VA_WIDTH_MASK          0x000000F0u
 207 +#define MMMU_DEBUG_INFO_PA_WIDTH_MASK          0x00000F00u
 208 +#define MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK     0x000FF000u
 209 +#define MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK   0x0FF00000u
 210 +#define MMMU_DEBUG_INFO_BYPASS_4M              BIT(28)
 211 +#define MMMU_DEBUG_INFO_BYPASS                 BIT(29)
 212 +
 213 +#define MMMU_PTE_PAGESIZE_MASK                 0xC0000000u
 214 +#define MMMU_PTE_WRITEABLE                     BIT(29)
 215 +#define MMMU_PTE_VALID                         BIT(28)
 216 +
 217 +/*
 218 + * BCM2712 IOMMU is organized around 4Kbyte pages (MMU_PAGE_SIZE).
 219 + * Linux PAGE_SIZE must not be smaller but may be larger (e.g. 4K, 16K).
 220 + *
 221 + * Unlike many larger MMUs, this one uses a 4-byte word size, allowing
 222 + * 1024 entries within each 4K table page, and two-level translation.
 223 + *
 224 + * Let's allocate enough table space for 2GB of translated memory (IOVA).
 225 + * This requires 512 4K pages (2MB) of level-2 tables, one page of
 226 + * top-level table (only half-filled in this particular configuration),
 227 + * plus one "default" page to catch illegal requests.
 228 + *
 229 + * The translated virtual address region is between 40GB and 42GB;
 230 + * addresses below this range pass straight through to the SDRAM.
 231 + *
 232 + * Currently we assume a 1:1:1 correspondence of IOMMU, group and domain.
 233 + */
 234 +
 235 +#define MMU_PAGE_SHIFT    12
 236 +#define MMU_PAGE_SIZE     BIT(MMU_PAGE_SHIFT)
 237 +
 238 +#define PAGEWORDS_SHIFT   (MMU_PAGE_SHIFT - 2)
 239 +#define HUGEPAGE_SHIFT    (MMU_PAGE_SHIFT + PAGEWORDS_SHIFT)
 240 +#define L1_CHUNK_SHIFT    (MMU_PAGE_SHIFT + 2 * PAGEWORDS_SHIFT)
 241 +
 242 +#define APERTURE_BASE     (40ul << 30)
 243 +#define APERTURE_SIZE     (2ul << 30)
 244 +#define APERTURE_TOP      (APERTURE_BASE + APERTURE_SIZE)
 245 +#define TRANSLATED_PAGES  (APERTURE_SIZE >> MMU_PAGE_SHIFT)
 246 +#define L2_PAGES          (TRANSLATED_PAGES >> PAGEWORDS_SHIFT)
 247 +#define TABLES_ALLOC_SIZE (L2_PAGES * MMU_PAGE_SIZE + 2 * PAGE_SIZE)
 248 +
 249 +static void bcm2712_iommu_init(struct bcm2712_iommu *mmu)
 250 +{
 251 +       unsigned int i, bypass_shift;
 252 +       struct sg_dma_page_iter it;
 253 +       u32 u = MMU_RD(MMMU_DEBUG_INFO_OFFSET);
 254 +
 255 +       /*
 256 +        * Check IOMMU version and hardware configuration.
 257 +        * This driver is for VC IOMMU version >= 4 (with 2-level tables)
 258 +        * and assumes at least 36 bits of virtual and physical address space.
 259 +        * Bigpage and superpage sizes are typically 64K and 1M, but may vary
 260 +        * (hugepage size is fixed at 4M, the range covered by an L2 page).
 261 +        */
 262 +       dev_info(mmu->dev, "%s: DEBUG_INFO = 0x%08x\n", __func__, u);
 263 +       WARN_ON(FIELD_GET(MMMU_DEBUG_INFO_VERSION_MASK, u) < 4 ||
 264 +               FIELD_GET(MMMU_DEBUG_INFO_VA_WIDTH_MASK, u) < 6 ||
 265 +               FIELD_GET(MMMU_DEBUG_INFO_PA_WIDTH_MASK, u) < 6 ||
 266 +               !(u & MMMU_DEBUG_INFO_BYPASS));
 267 +
 268 +       mmu->bigpage_mask =
 269 +               ((1u << FIELD_GET(MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK, u)) - 1u) << MMU_PAGE_SHIFT;
 270 +       mmu->superpage_mask =
 271 +               ((1u << FIELD_GET(MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK, u)) - 1u) << MMU_PAGE_SHIFT;
 272 +       bypass_shift = (u & MMMU_DEBUG_INFO_BYPASS_4M) ?
 273 +               HUGEPAGE_SHIFT : ADDR_CAP_SHIFT;
 274 +
 275 +       /* Disable MMU and clear sticky flags; meanwhile flush the TLB */
 276 +       MMU_WR(MMMU_CTRL_OFFSET,
 277 +              MMMU_CTRL_CAP_EXCEEDED    |
 278 +              MMMU_CTRL_PT_INVALID      |
 279 +              MMMU_CTRL_WRITE_VIOLATION |
 280 +              MMMU_CTRL_STATS_CLEAR     |
 281 +              MMMU_CTRL_TLB_CLEAR);
 282 +
 283 +       /*
 284 +        * Put MMU into 2-level mode; set address cap and "bypass" range
 285 +        * (note that some of these registers have unintuitive off-by-ones).
 286 +        * Addresses below APERTURE_BASE are passed unchanged: this is
 287 +        * useful for blocks which share an IOMMU with other blocks
 288 +        * whose drivers are not IOMMU-aware.
 289 +        */
 290 +       MMU_WR(MMMU_MISC_OFFSET,
 291 +              MMU_RD(MMMU_MISC_OFFSET) & ~MMMU_MISC_SINGLE_TABLE);
 292 +       MMU_WR(MMMU_ADDR_CAP_OFFSET,
 293 +              MMMU_ADDR_CAP_ENABLE +
 294 +              (APERTURE_TOP >> ADDR_CAP_SHIFT) - 1);
 295 +       if (APERTURE_BASE > 0) {
 296 +               MMU_WR(MMMU_BYPASS_START_OFFSET,
 297 +                      MMMU_BYPASS_START_ENABLE + MMMU_BYPASS_START_INVERT +
 298 +                      (APERTURE_BASE >> bypass_shift) - 1);
 299 +               MMU_WR(MMMU_BYPASS_END_OFFSET,
 300 +                      MMMU_BYPASS_END_ENABLE +
 301 +                      (APERTURE_TOP >> bypass_shift));
 302 +       } else {
 303 +               MMU_WR(MMMU_BYPASS_START_OFFSET, 0);
 304 +               MMU_WR(MMMU_BYPASS_END_OFFSET, 0);
 305 +       }
 306 +
 307 +       /* Ensure tables are zeroed (which marks all pages as invalid) */
 308 +       dma_sync_sgtable_for_cpu(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
 309 +       memset(mmu->tables, 0, TABLES_ALLOC_SIZE);
 310 +       mmu->nmapped_pages = 0;
 311 +
 312 +       /* Initialize the high-level table to point to the low-level pages */
 313 +       __sg_page_iter_start(&it.base, mmu->sgt->sgl, mmu->sgt->nents, 0);
 314 +       for (i = 0; i < L2_PAGES; i++) {
 315 +               if (!(i % (PAGE_SIZE / MMU_PAGE_SIZE))) {
 316 +                       __sg_page_iter_dma_next(&it);
 317 +                       u = (sg_page_iter_dma_address(&it) >> MMU_PAGE_SHIFT);
 318 +               } else {
 319 +                       u++;
 320 +               }
 321 +               mmu->tables[TRANSLATED_PAGES + i] = MMMU_PTE_VALID + u;
 322 +       }
 323 +
 324 +       /*
 325 +        * Configure the addresses of the top-level table (offset because
 326 +        * the aperture does not start from zero), and of the default page.
 327 +        * For simplicity, both these regions are whole Linux pages.
 328 +        */
 329 +       __sg_page_iter_dma_next(&it);
 330 +       u = (sg_page_iter_dma_address(&it) >> MMU_PAGE_SHIFT);
 331 +       MMU_WR(MMMU_PT_PA_BASE_OFFSET, u - (APERTURE_BASE >> L1_CHUNK_SHIFT));
 332 +       __sg_page_iter_dma_next(&it);
 333 +       u = (sg_page_iter_dma_address(&it) >> MMU_PAGE_SHIFT);
 334 +       MMU_WR(MMMU_ILLEGAL_ADR_OFFSET, MMMU_ILLEGAL_ADR_ENABLE + u);
 335 +       dma_sync_sgtable_for_device(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
 336 +       mmu->dirty = false;
 337 +
 338 +       /* Flush (and enable) the shared TLB cache; enable this MMU. */
 339 +       if (mmu->cache)
 340 +               bcm2712_iommu_cache_flush(mmu->cache);
 341 +       MMU_WR(MMMU_CTRL_OFFSET,
 342 +              MMMU_CTRL_CAP_EXCEEDED_ABORT_EN    |
 343 +              MMMU_CTRL_PT_INVALID_ABORT_EN      |
 344 +              MMMU_CTRL_WRITE_VIOLATION_ABORT_EN |
 345 +              MMMU_CTRL_STATS_ENABLE             |
 346 +              MMMU_CTRL_ENABLE);
 347 +}
 348 +
 349 +static int bcm2712_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
 350 +{
 351 +       struct bcm2712_iommu *mmu = dev ? dev_iommu_priv_get(dev) : 0;
 352 +       struct bcm2712_iommu_domain *mydomain =
 353 +               container_of(domain, struct bcm2712_iommu_domain, base);
 354 +
 355 +       dev_info(dev, "%s: MMU %s\n",
 356 +                __func__, mmu ? dev_name(mmu->dev) : "");
 357 +
 358 +       if (mmu) {
 359 +               mydomain->mmu = mmu;
 360 +               mmu->domain = mydomain;
 361 +
 362 +               if (mmu->dma_iova_offset) {
 363 +                       domain->geometry.aperture_start =
 364 +                               mmu->dma_iova_offset + APERTURE_BASE;
 365 +                       domain->geometry.aperture_end =
 366 +                               mmu->dma_iova_offset + APERTURE_TOP - 1ul;
 367 +               }
 368 +
 369 +               return 0;
 370 +       }
 371 +       return -EINVAL;
 372 +}
 373 +
 374 +static void bcm2712_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
 375 +{
 376 +       (void)domain;
 377 +       (void)dev;
 378 +}
 379 +
 380 +static int bcm2712_iommu_map(struct iommu_domain *domain, unsigned long iova,
 381 +                            phys_addr_t pa, size_t bytes, int prot, gfp_t gfp)
 382 +{
 383 +       struct bcm2712_iommu *mmu = domain_to_mmu(domain);
 384 +
 385 +       (void)gfp;
 386 +       iova -= mmu->dma_iova_offset;
 387 +       if (iova >= APERTURE_BASE && iova + bytes <= APERTURE_TOP) {
 388 +               unsigned int p;
 389 +               u32 entry = MMMU_PTE_VALID | (pa >> MMU_PAGE_SHIFT);
 390 +               u32 align = (u32)(iova | pa | bytes);
 391 +
 392 +               /* large page and write enable flags */
 393 +               if (!(align & ((1 << HUGEPAGE_SHIFT) - 1)))
 394 +                       entry |= FIELD_PREP(MMMU_PTE_PAGESIZE_MASK, 3);
 395 +               else if (!(align & mmu->superpage_mask) && mmu->superpage_mask)
 396 +                       entry |= FIELD_PREP(MMMU_PTE_PAGESIZE_MASK, 2);
 397 +               else if (!(align &  mmu->bigpage_mask) && mmu->bigpage_mask)
 398 +                       entry |= FIELD_PREP(MMMU_PTE_PAGESIZE_MASK, 1);
 399 +               if (prot & IOMMU_WRITE)
 400 +                       entry |= MMMU_PTE_WRITEABLE;
 401 +
 402 +               /* Ensure tables are cache-coherent with CPU */
 403 +               if (!mmu->dirty) {
 404 +                       dma_sync_sgtable_for_cpu(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
 405 +                       mmu->dirty = true;
 406 +               }
 407 +
 408 +               iova -= APERTURE_BASE;
 409 +               for (p = iova >> MMU_PAGE_SHIFT;
 410 +                    p < (iova + bytes) >> MMU_PAGE_SHIFT; p++) {
 411 +                       mmu->nmapped_pages += !(mmu->tables[p]);
 412 +                       mmu->tables[p] = entry++;
 413 +               }
 414 +       } else if (iova + bytes > APERTURE_BASE || iova != pa) {
 415 +               dev_warn(mmu->dev, "%s: iova=0x%lx pa=0x%llx size=0x%llx OUT OF RANGE!\n",
 416 +                        __func__, iova,
 417 +                        (unsigned long long)pa, (unsigned long long)bytes);
 418 +               return -EINVAL;
 419 +       }
 420 +
 421 +       return 0;
 422 +}
 423 +
 424 +static size_t bcm2712_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
 425 +                                 size_t bytes, struct iommu_iotlb_gather *gather)
 426 +{
 427 +       struct bcm2712_iommu *mmu = domain_to_mmu(domain);
 428 +
 429 +       if (iova >= mmu->dma_iova_offset + APERTURE_BASE &&
 430 +           iova + bytes <= mmu->dma_iova_offset + APERTURE_TOP) {
 431 +               unsigned int p;
 432 +
 433 +               /* Record just the lower and upper bounds in "gather" */
 434 +               if (gather) {
 435 +                       bool empty = (gather->end <= gather->start);
 436 +
 437 +                       if (empty || gather->start < iova)
 438 +                               gather->start = iova;
 439 +                       if (empty || gather->end < iova + bytes)
 440 +                               gather->end = iova + bytes;
 441 +               }
 442 +
 443 +               /* Ensure tables are cache-coherent with CPU */
 444 +               if (!mmu->dirty) {
 445 +                       dma_sync_sgtable_for_cpu(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
 446 +                       mmu->dirty = true;
 447 +               }
 448 +
 449 +               /* Clear table entries, this marks the addresses as illegal */
 450 +               iova -= (mmu->dma_iova_offset + APERTURE_BASE);
 451 +               for (p = iova >> MMU_PAGE_SHIFT;
 452 +                    p < (iova + bytes) >> MMU_PAGE_SHIFT;
 453 +                    p++) {
 454 +                       mmu->nmapped_pages -= !!(mmu->tables[p]);
 455 +                       mmu->tables[p] = 0;
 456 +               }
 457 +       }
 458 +
 459 +       return bytes;
 460 +}
 461 +
 462 +static void bcm2712_iommu_sync_range(struct iommu_domain *domain,
 463 +                                    unsigned long iova, size_t size)
 464 +{
 465 +       struct bcm2712_iommu *mmu = domain_to_mmu(domain);
 466 +       unsigned long iova_end;
 467 +       unsigned int i, p4;
 468 +
 469 +       if (!mmu || !mmu->dirty)
 470 +               return;
 471 +
 472 +       /* Ensure tables are cleaned from CPU cache or write-buffer */
 473 +       dma_sync_sgtable_for_device(mmu->dev, mmu->sgt, DMA_TO_DEVICE);
 474 +       mmu->dirty = false;
 475 +
 476 +       /* Flush the shared TLB cache */
 477 +       if (mmu->cache)
 478 +               bcm2712_iommu_cache_flush(mmu->cache);
 479 +
 480 +       /*
 481 +        * When flushing a large range or when nothing needs to be kept,
 482 +        * it's quicker to use the"TLB_CLEAR" flag. Otherwise, invalidate
 483 +        * TLB entries in lines of 4 words each. Each flush/clear operation
 484 +        * should complete almost instantaneously.
 485 +        */
 486 +       iova -= mmu->dma_iova_offset;
 487 +       iova_end = min(APERTURE_TOP, iova + size);
 488 +       iova = max(APERTURE_BASE, iova);
 489 +       if (mmu->nmapped_pages == 0 || iova_end - iova >= APERTURE_SIZE / 8) {
 490 +               MMU_WR(MMMU_CTRL_OFFSET,
 491 +                      MMMU_CTRL_CAP_EXCEEDED_ABORT_EN    |
 492 +                      MMMU_CTRL_PT_INVALID_ABORT_EN      |
 493 +                      MMMU_CTRL_WRITE_VIOLATION_ABORT_EN |
 494 +                      MMMU_CTRL_TLB_CLEAR                |
 495 +                      MMMU_CTRL_STATS_ENABLE             |
 496 +                      MMMU_CTRL_ENABLE);
 497 +               for (i = 0; i < 1024; i++) {
 498 +                       if (!(MMMU_CTRL_TLB_CLEARING & MMU_RD(MMMU_CTRL_OFFSET)))
 499 +                               break;
 500 +                       cpu_relax();
 501 +               }
 502 +       } else {
 503 +               for (p4 = iova >> (MMU_PAGE_SHIFT + 2);
 504 +                    p4 < (iova_end + 3 * MMU_PAGE_SIZE) >> (MMU_PAGE_SHIFT + 2);
 505 +                    p4++) {
 506 +                       MMU_WR(MMMU_SHOOT_DOWN_OFFSET,
 507 +                              MMMU_SHOOT_DOWN_SHOOT + (p4 << 2));
 508 +                       for (i = 0; i < 1024; i++) {
 509 +                               if (!(MMMU_SHOOT_DOWN_SHOOTING & MMU_RD(MMMU_SHOOT_DOWN_OFFSET)))
 510 +                                       break;
 511 +                               cpu_relax();
 512 +                       }
 513 +               }
 514 +       }
 515 +}
 516 +
 517 +static void bcm2712_iommu_sync(struct iommu_domain *domain,
 518 +                              struct iommu_iotlb_gather *gather)
 519 +{
 520 +       bcm2712_iommu_sync_range(domain, gather->start,
 521 +                                gather->end - gather->start);
 522 +}
 523 +
 524 +static void bcm2712_iommu_sync_all(struct iommu_domain *domain)
 525 +{
 526 +       bcm2712_iommu_sync_range(domain, APERTURE_BASE, APERTURE_SIZE);
 527 +}
 528 +
 529 +static phys_addr_t bcm2712_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
 530 +{
 531 +       struct bcm2712_iommu *mmu = domain_to_mmu(domain);
 532 +       u32 p;
 533 +
 534 +       iova -= mmu->dma_iova_offset;
 535 +       if (iova  >= APERTURE_BASE && iova < APERTURE_TOP) {
 536 +               p = (iova - APERTURE_BASE) >> MMU_PAGE_SHIFT;
 537 +               p = mmu->tables[p] & 0x0FFFFFFFu;
 538 +               return (((phys_addr_t)p) << MMU_PAGE_SHIFT) + (iova & (MMU_PAGE_SIZE - 1u));
 539 +       } else if (iova < APERTURE_BASE) {
 540 +               return (phys_addr_t)iova;
 541 +       } else {
 542 +               return (phys_addr_t)-EINVAL;
 543 +       }
 544 +}
 545 +
 546 +static void bcm2712_iommu_domain_free(struct iommu_domain *domain)
 547 +{
 548 +       struct bcm2712_iommu_domain *mydomain =
 549 +               container_of(domain, struct bcm2712_iommu_domain, base);
 550 +
 551 +       kfree(mydomain);
 552 +}
 553 +
 554 +static const struct iommu_domain_ops bcm2712_iommu_domain_ops = {
 555 +       .attach_dev      = bcm2712_iommu_attach_dev,
 556 +       .detach_dev      = bcm2712_iommu_detach_dev,
 557 +       .map             = bcm2712_iommu_map,
 558 +       .unmap           = bcm2712_iommu_unmap,
 559 +       .iotlb_sync      = bcm2712_iommu_sync,
 560 +       .iotlb_sync_map  = bcm2712_iommu_sync_range,
 561 +       .flush_iotlb_all = bcm2712_iommu_sync_all,
 562 +       .iova_to_phys    = bcm2712_iommu_iova_to_phys,
 563 +       .free            = bcm2712_iommu_domain_free,
 564 +};
 565 +
 566 +static struct iommu_domain *bcm2712_iommu_domain_alloc(unsigned int type)
 567 +{
 568 +       struct bcm2712_iommu_domain *domain;
 569 +
 570 +       if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
 571 +               return NULL;
 572 +
 573 +       domain = kzalloc(sizeof(*domain), GFP_KERNEL);
 574 +       if (!domain)
 575 +               return NULL;
 576 +
 577 +       domain->base.type = type;
 578 +       domain->base.ops  = &bcm2712_iommu_domain_ops;
 579 +       domain->base.geometry.aperture_start = APERTURE_BASE;
 580 +       domain->base.geometry.aperture_end   = APERTURE_TOP - 1ul;
 581 +       domain->base.geometry.force_aperture = true;
 582 +       return &domain->base;
 583 +}
 584 +
 585 +static struct iommu_device *bcm2712_iommu_probe_device(struct device *dev)
 586 +{
 587 +       struct bcm2712_iommu *mmu;
 588 +
 589 +       /*
 590 +        * For reasons I don't fully understand, we need to try both
 591 +        * cases (dev_iommu_priv_get() and platform_get_drvdata())
 592 +        * in order to get both GPU and ISP-BE to probe successfully.
 593 +        */
 594 +       mmu = dev_iommu_priv_get(dev);
 595 +       if (!mmu) {
 596 +               struct device_node *np;
 597 +               struct platform_device *pdev;
 598 +
 599 +               /* Ignore devices that don't have an "iommus" property with exactly one phandle */
 600 +               if (!dev->of_node ||
 601 +                   of_property_count_elems_of_size(dev->of_node, "iommus", sizeof(phandle)) != 1)
 602 +                       return ERR_PTR(-ENODEV);
 603 +
 604 +               np = of_parse_phandle(dev->of_node, "iommus", 0);
 605 +               if (!np)
 606 +                       return ERR_PTR(-EINVAL);
 607 +
 608 +               pdev = of_find_device_by_node(np);
 609 +               of_node_put(np);
 610 +               if (pdev)
 611 +                       mmu = platform_get_drvdata(pdev);
 612 +
 613 +               if (!mmu)
 614 +                       return ERR_PTR(-ENODEV);
 615 +       }
 616 +
 617 +       dev_info(dev, "%s: MMU %s\n", __func__, dev_name(mmu->dev));
 618 +       dev_iommu_priv_set(dev, mmu);
 619 +       return &mmu->iommu;
 620 +}
 621 +
 622 +static void bcm2712_iommu_release_device(struct device *dev)
 623 +{
 624 +       dev_iommu_priv_set(dev, NULL);
 625 +}
 626 +
 627 +static struct iommu_group *bcm2712_iommu_device_group(struct device *dev)
 628 +{
 629 +       struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
 630 +
 631 +       if (!mmu || !mmu->group)
 632 +               return ERR_PTR(-EINVAL);
 633 +
 634 +       dev_info(dev, "%s: MMU %s\n", __func__, dev_name(mmu->dev));
 635 +       return iommu_group_ref_get(mmu->group);
 636 +}
 637 +
 638 +static int bcm2712_iommu_of_xlate(struct device *dev,
 639 +                                 struct of_phandle_args *args)
 640 +{
 641 +       struct platform_device *iommu_dev;
 642 +       struct bcm2712_iommu *mmu;
 643 +
 644 +       iommu_dev = of_find_device_by_node(args->np);
 645 +       mmu = platform_get_drvdata(iommu_dev);
 646 +       dev_iommu_priv_set(dev, mmu);
 647 +       dev_info(dev, "%s: MMU %s\n", __func__, dev_name(mmu->dev));
 648 +
 649 +       return 0;
 650 +}
 651 +
 652 +static bool bcm2712_iommu_capable(struct device *dev, enum iommu_cap cap)
 653 +{
 654 +       return false;
 655 +}
 656 +
 657 +static const struct iommu_ops bcm2712_iommu_ops = {
 658 +       .capable        = bcm2712_iommu_capable,
 659 +       .domain_alloc   = bcm2712_iommu_domain_alloc,
 660 +       .probe_device   = bcm2712_iommu_probe_device,
 661 +       .release_device = bcm2712_iommu_release_device,
 662 +       .device_group   = bcm2712_iommu_device_group,
 663 +       /* Advertise native page sizes as well as 2M, 16K which Linux may prefer */
 664 +       .pgsize_bitmap  = (SZ_4M | SZ_2M | SZ_1M | SZ_64K | SZ_16K | SZ_4K),
 665 +       .default_domain_ops = &bcm2712_iommu_domain_ops,
 666 +       .of_xlate = bcm2712_iommu_of_xlate,
 667 +};
 668 +
 669 +static int bcm2712_iommu_probe(struct platform_device *pdev)
 670 +{
 671 +       struct bcm2712_iommu *mmu;
 672 +       struct bcm2712_iommu_cache *cache = NULL;
 673 +       int ret;
 674 +
 675 +       /* First of all, check for an IOMMU shared cache */
 676 +       if (pdev->dev.of_node) {
 677 +               struct device_node *cache_np;
 678 +               struct platform_device *cache_pdev;
 679 +
 680 +               cache_np = of_parse_phandle(pdev->dev.of_node, "cache", 0);
 681 +               if (cache_np) {
 682 +                       cache_pdev = of_find_device_by_node(cache_np);
 683 +                       of_node_put(cache_np);
 684 +                       if (cache_pdev && !IS_ERR(cache_pdev))
 685 +                               cache = platform_get_drvdata(cache_pdev);
 686 +                       if (!cache)
 687 +                               return -EPROBE_DEFER;
 688 +               }
 689 +       }
 690 +
 691 +       /* Allocate private data */
 692 +       mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);
 693 +       if (!mmu)
 694 +               return -ENOMEM;
 695 +
 696 +       mmu->name = dev_name(&pdev->dev);
 697 +       mmu->dev = &pdev->dev;
 698 +       mmu->cache = cache;
 699 +       platform_set_drvdata(pdev, mmu);
 700 +       spin_lock_init(&mmu->hw_lock);
 701 +
 702 +       /*
 703 +        * XXX When an IOMMU is downstream of a PCIe RC or some other chip/bus
 704 +        * and serves some of the masters thereon (others using pass-through),
 705 +        * we seem to fumble and lose the "dma-ranges" address offset for
 706 +        * masters using IOMMU. This property restores it, where needed.
 707 +        */
 708 +       if (!pdev->dev.of_node ||
 709 +           of_property_read_u64(pdev->dev.of_node, "dma-iova-offset",
 710 +                                &mmu->dma_iova_offset))
 711 +               mmu->dma_iova_offset = 0;
 712 +
 713 +       /*
 714 +        * The IOMMU is itself a device that allocates DMA-able memory
 715 +        * to hold its translation tables. Provided the IOVA aperture
 716 +        * is no larger than 4 GBytes (so that the L1 table fits within
 717 +        * a single 4K page), we don't need the tables to be contiguous.
 718 +        * Assume we can address at least 36 bits (64 GB).
 719 +        */
 720 +       ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36));
 721 +       WARN_ON(ret);
 722 +       mmu->sgt = dma_alloc_noncontiguous(&pdev->dev, TABLES_ALLOC_SIZE,
 723 +                                          DMA_TO_DEVICE, GFP_KERNEL,
 724 +                                          DMA_ATTR_ALLOC_SINGLE_PAGES);
 725 +       if (!mmu->sgt) {
 726 +               ret = -ENOMEM;
 727 +               goto done_err;
 728 +       }
 729 +       mmu->tables = dma_vmap_noncontiguous(&pdev->dev, TABLES_ALLOC_SIZE,
 730 +                                            mmu->sgt);
 731 +       if (!mmu->tables) {
 732 +               ret = -ENOMEM;
 733 +               goto done_err;
 734 +       }
 735 +
 736 +       /* Get IOMMU registers */
 737 +       mmu->reg_base = devm_platform_ioremap_resource(pdev, 0);
 738 +       if (IS_ERR(mmu->reg_base)) {
 739 +               dev_err(&pdev->dev, "Failed to get IOMMU registers address\n");
 740 +               ret = PTR_ERR(mmu->reg_base);
 741 +               goto done_err;
 742 +       }
 743 +
 744 +       /* Stuff */
 745 +       mmu->group = iommu_group_alloc();
 746 +       if (IS_ERR(mmu->group)) {
 747 +               ret = PTR_ERR(mmu->group);
 748 +               mmu->group = NULL;
 749 +               goto done_err;
 750 +       }
 751 +       ret = iommu_device_sysfs_add(&mmu->iommu, mmu->dev, NULL, mmu->name);
 752 +       if (ret)
 753 +               goto done_err;
 754 +
 755 +       /* Initialize table and hardware */
 756 +       bcm2712_iommu_init(mmu);
 757 +       ret = iommu_device_register(&mmu->iommu, &bcm2712_iommu_ops, &pdev->dev);
 758 +
 759 +       dev_info(&pdev->dev, "%s: Success\n", __func__);
 760 +       return 0;
 761 +
 762 +done_err:
 763 +       dev_info(&pdev->dev, "%s: Failure %d\n", __func__, ret);
 764 +       if (mmu->group)
 765 +               iommu_group_put(mmu->group);
 766 +       if (mmu->tables)
 767 +               dma_vunmap_noncontiguous(&pdev->dev,
 768 +                                        (void *)(mmu->tables));
 769 +       mmu->tables = NULL;
 770 +       if (mmu->sgt)
 771 +               dma_free_noncontiguous(&pdev->dev, TABLES_ALLOC_SIZE,
 772 +                                      mmu->sgt, DMA_TO_DEVICE);
 773 +       mmu->sgt = NULL;
 774 +       kfree(mmu);
 775 +       return ret;
 776 +}
 777 +
 778 +static int bcm2712_iommu_remove(struct platform_device *pdev)
 779 +{
 780 +       struct bcm2712_iommu *mmu = platform_get_drvdata(pdev);
 781 +
 782 +       if (mmu->reg_base)
 783 +               MMU_WR(MMMU_CTRL_OFFSET, 0); /* disable the MMU */
 784 +       if (mmu->sgt)
 785 +               dma_free_noncontiguous(&pdev->dev, TABLES_ALLOC_SIZE,
 786 +                                      mmu->sgt, DMA_TO_DEVICE);
 787 +
 788 +       return 0;
 789 +}
 790 +
 791 +static const struct of_device_id bcm2712_iommu_of_match[] = {
 792 +       {
 793 +               . compatible = "brcm,bcm2712-iommu"
 794 +       },
 795 +       { /* sentinel */ },
 796 +};
 797 +
 798 +static struct platform_driver bcm2712_iommu_driver = {
 799 +       .probe = bcm2712_iommu_probe,
 800 +       .remove = bcm2712_iommu_remove,
 801 +       .driver = {
 802 +               .name = "bcm2712-iommu",
 803 +               .of_match_table = bcm2712_iommu_of_match
 804 +       },
 805 +};
 806 +
 807 +builtin_platform_driver(bcm2712_iommu_driver);
 808 --- /dev/null
 809 +++ b/drivers/iommu/bcm2712-iommu.h
 810 @@ -0,0 +1,45 @@
 811 +/* SPDX-License-Identifier: GPL-2.0-only */
 812 +/*
 813 + * IOMMU driver for BCM2712
 814 + *
 815 + * Copyright (c) 2023 Raspberry Pi Ltd.
 816 + */
 817 +
 818 +#ifndef _BCM2712_IOMMU_H
 819 +#define _BCM2712_IOMMU_H
 820 +
 821 +#include <linux/iommu.h>
 822 +#include <linux/scatterlist.h>
 823 +
 824 +struct bcm2712_iommu_cache {
 825 +       struct device *dev;
 826 +       spinlock_t hw_lock; /* to protect HW registers */
 827 +       void __iomem *reg_base;
 828 +};
 829 +
 830 +void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache);
 831 +
 832 +struct bcm2712_iommu {
 833 +       struct device *dev;
 834 +       struct iommu_device iommu;
 835 +       struct iommu_group *group;
 836 +       struct bcm2712_iommu_domain *domain;
 837 +       char const *name;
 838 +       struct sg_table *sgt; /* allocated memory for page tables */
 839 +       u32 *tables;          /* kernel mapping for page tables */
 840 +       struct bcm2712_iommu_cache *cache;
 841 +       spinlock_t hw_lock;   /* to protect HW registers */
 842 +       void __iomem *reg_base;
 843 +       u64 dma_iova_offset; /* Hack for IOMMU attached to PCIe RC */
 844 +       u32 bigpage_mask;
 845 +       u32 superpage_mask;
 846 +       unsigned int nmapped_pages;
 847 +       bool dirty; /* true when tables are oriented towards CPU */
 848 +};
 849 +
 850 +struct bcm2712_iommu_domain {
 851 +       struct iommu_domain base;
 852 +       struct bcm2712_iommu *mmu;
 853 +};
 854 +
 855 +#endif