target/linux/brcm2708/patches-4.14/950-0156-vcsm-Revert-to-do-page-table-walk-based-cache-manipu.patch

   1 From 3e471b58fb766ed5b45ff8b560231da4704c946d Mon Sep 17 00:00:00 2001
   2 From: Sugizaki Yukimasa <i.can.speak.c.and.basic@gmail.com>
   3 Date: Wed, 10 Jan 2018 06:25:51 +0900
   4 Subject: [PATCH 156/454] vcsm: Revert to do page-table-walk-based cache
   5  manipulating on some ioctl calls
   6
   7 On FLUSH, INVALID, CLEAN_INVALID ioctl calls, cache operations based on
   8 page table walk were used in case that the buffer of the cache is not
   9 pinned.  So reverted to do page-table-based cache manipulating.
  10
  11 Signed-off-by: Sugizaki Yukimasa <i.can.speak.c.and.basic@gmail.com>
  12 ---
  13  drivers/char/broadcom/vc_sm/vmcs_sm.c | 141 ++++++++++++++++++++------
  14  1 file changed, 110 insertions(+), 31 deletions(-)
  15
  16 --- a/drivers/char/broadcom/vc_sm/vmcs_sm.c
  17 +++ b/drivers/char/broadcom/vc_sm/vmcs_sm.c
  18 @@ -1256,7 +1256,33 @@ static const struct vm_operations_struct
  19         .fault = vcsm_vma_fault,
  20  };
  21
  22 -static int clean_invalid_mem_2d(const void __user *addr,
  23 +/* Converts VCSM_CACHE_OP_* to an operating function. */
  24 +static void (*cache_op_to_func(const unsigned cache_op))
  25 +               (const void*, const void*)
  26 +{
  27 +       switch (cache_op) {
  28 +       case VCSM_CACHE_OP_NOP:
  29 +               return NULL;
  30 +
  31 +       case VCSM_CACHE_OP_INV:
  32 +               return dmac_inv_range;
  33 +
  34 +       case VCSM_CACHE_OP_CLEAN:
  35 +               return dmac_clean_range;
  36 +
  37 +       case VCSM_CACHE_OP_FLUSH:
  38 +               return dmac_flush_range;
  39 +
  40 +       default:
  41 +               pr_err("[%s]: Invalid cache_op: 0x%08x\n", __func__, cache_op);
  42 +               return NULL;
  43 +       }
  44 +}
  45 +
  46 +/*
  47 + * Clean/invalid/flush cache of which buffer is already pinned (i.e. accessed).
  48 + */
  49 +static int clean_invalid_contiguous_mem_2d(const void __user *addr,
  50                 const size_t block_count, const size_t block_size, const size_t stride,
  51                 const unsigned cache_op)
  52  {
  53 @@ -1268,22 +1294,9 @@ static int clean_invalid_mem_2d(const vo
  54                 return -EINVAL;
  55         }
  56
  57 -       switch (cache_op) {
  58 -       case VCSM_CACHE_OP_NOP:
  59 -               return 0;
  60 -       case VCSM_CACHE_OP_INV:
  61 -               op_fn = dmac_inv_range;
  62 -               break;
  63 -       case VCSM_CACHE_OP_CLEAN:
  64 -               op_fn = dmac_clean_range;
  65 -               break;
  66 -       case VCSM_CACHE_OP_FLUSH:
  67 -               op_fn = dmac_flush_range;
  68 -               break;
  69 -       default:
  70 -               pr_err("[%s]: Invalid cache_op: 0x%08x\n", __func__, cache_op);
  71 +       op_fn = cache_op_to_func(cache_op);
  72 +       if (op_fn == NULL)
  73                 return -EINVAL;
  74 -       }
  75
  76         for (i = 0; i < block_count; i ++, addr += stride)
  77                 op_fn(addr, addr + block_size);
  78 @@ -1291,14 +1304,73 @@ static int clean_invalid_mem_2d(const vo
  79         return 0;
  80  }
  81
  82 -static int clean_invalid_mem(const void __user *addr, const size_t size,
  83 +/* Clean/invalid/flush cache of which buffer may be non-pinned. */
  84 +/* The caller must lock current->mm->mmap_sem for read. */
  85 +static int clean_invalid_mem_walk(unsigned long addr, const size_t size,
  86                 const unsigned cache_op)
  87  {
  88 -       return clean_invalid_mem_2d(addr, 1, size, 0, cache_op);
  89 +       pgd_t *pgd;
  90 +       pud_t *pud;
  91 +       pmd_t *pmd;
  92 +       pte_t *pte;
  93 +       unsigned long pgd_next, pud_next, pmd_next;
  94 +       const unsigned long end = ALIGN(addr + size, PAGE_SIZE);
  95 +       void (*op_fn)(const void*, const void*);
  96 +
  97 +       addr &= PAGE_MASK;
  98 +
  99 +       if (addr >= end)
 100 +               return 0;
 101 +
 102 +       op_fn = cache_op_to_func(cache_op);
 103 +       if (op_fn == NULL)
 104 +               return -EINVAL;
 105 +
 106 +       /* Walk PGD */
 107 +       pgd = pgd_offset(current->mm, addr);
 108 +       do {
 109 +               pgd_next = pgd_addr_end(addr, end);
 110 +
 111 +               if (pgd_none(*pgd) || pgd_bad(*pgd))
 112 +                       continue;
 113 +
 114 +               /* Walk PUD */
 115 +               pud = pud_offset(pgd, addr);
 116 +               do {
 117 +                       pud_next = pud_addr_end(addr, pgd_next);
 118 +                       if (pud_none(*pud) || pud_bad(*pud))
 119 +                               continue;
 120 +
 121 +                       /* Walk PMD */
 122 +                       pmd = pmd_offset(pud, addr);
 123 +                       do {
 124 +                               pmd_next = pmd_addr_end(addr, pud_next);
 125 +                               if (pmd_none(*pmd) || pmd_bad(*pmd))
 126 +                                       continue;
 127 +
 128 +                               /* Walk PTE */
 129 +                               pte = pte_offset_map(pmd, addr);
 130 +                               do {
 131 +                                       if (pte_none(*pte) || !pte_present(*pte))
 132 +                                               continue;
 133 +
 134 +                                       op_fn((const void __user*) addr,
 135 +                                                       (const void __user*) (addr + PAGE_SIZE));
 136 +                               } while (pte++, addr += PAGE_SIZE, addr != pmd_next);
 137 +                               pte_unmap(pte);
 138 +
 139 +                       } while (pmd++, addr = pmd_next, addr != pud_next);
 140 +
 141 +               } while (pud++, addr = pud_next, addr != pgd_next);
 142 +
 143 +       } while (pgd++, addr = pgd_next, addr != end);
 144 +
 145 +       return 0;
 146  }
 147
 148 -static int clean_invalid_resource(const void __user *addr, const size_t size,
 149 -               const unsigned cache_op, const int usr_hdl,
 150 +/* Clean/invalid/flush cache of buffer in resource */
 151 +static int clean_invalid_resource_walk(const void __user *addr,
 152 +               const size_t size, const unsigned cache_op, const int usr_hdl,
 153                 struct sm_resource_t *resource)
 154  {
 155         int err;
 156 @@ -1355,7 +1427,10 @@ static int clean_invalid_resource(const
 157                 return -EFAULT;
 158         }
 159
 160 -       err = clean_invalid_mem(addr, size, cache_op);
 161 +       down_read(&current->mm->mmap_sem);
 162 +       err = clean_invalid_mem_walk((unsigned long) addr, size, cache_op);
 163 +       up_read(&current->mm->mmap_sem);
 164 +
 165         if (err)
 166                 resource->res_stats[stat_failure]++;
 167
 168 @@ -2004,7 +2079,7 @@ static int vc_sm_ioctl_unlock(struct sm_
 169                                         const unsigned long start = map->vma->vm_start;
 170                                         const unsigned long end = map->vma->vm_end;
 171
 172 -                                       ret = clean_invalid_mem((void __user*) start, end - start,
 173 +                                       ret = clean_invalid_mem_walk(start, end - start,
 174                                                         VCSM_CACHE_OP_FLUSH);
 175                                         if (ret)
 176                                                 goto error;
 177 @@ -2886,7 +2961,7 @@ static long vc_sm_ioctl(struct file *fil
 178                                 goto out;
 179                         }
 180
 181 -                       ret = clean_invalid_resource((void __user*) ioparam.addr,
 182 +                       ret = clean_invalid_resource_walk((void __user*) ioparam.addr,
 183                                         ioparam.size, VCSM_CACHE_OP_FLUSH, ioparam.handle,
 184                                         resource);
 185                         vmcs_sm_release_resource(resource, 0);
 186 @@ -2917,7 +2992,7 @@ static long vc_sm_ioctl(struct file *fil
 187                                 goto out;
 188                         }
 189
 190 -                       ret = clean_invalid_resource((void __user*) ioparam.addr,
 191 +                       ret = clean_invalid_resource_walk((void __user*) ioparam.addr,
 192                                         ioparam.size, VCSM_CACHE_OP_INV, ioparam.handle, resource);
 193                         vmcs_sm_release_resource(resource, 0);
 194                         if (ret)
 195 @@ -2951,16 +3026,19 @@ static long vc_sm_ioctl(struct file *fil
 196                                         goto out;
 197                                 }
 198
 199 -                               ret = clean_invalid_resource((void __user*) ioparam.s[i].addr,
 200 -                                               ioparam.s[i].size, ioparam.s[i].cmd,
 201 -                                               ioparam.s[i].handle, resource);
 202 +                               ret = clean_invalid_resource_walk(
 203 +                                               (void __user*) ioparam.s[i].addr, ioparam.s[i].size,
 204 +                                               ioparam.s[i].cmd, ioparam.s[i].handle, resource);
 205                                 vmcs_sm_release_resource(resource, 0);
 206                                 if (ret)
 207                                         goto out;
 208                         }
 209                 }
 210                 break;
 211 -       /* Flush/Invalidate the cache for a given mapping. */
 212 +       /*
 213 +        * Flush/Invalidate the cache for a given mapping.
 214 +        * Blocks must be pinned (i.e. accessed) before this call.
 215 +        */
 216         case VMCS_SM_CMD_CLEAN_INVALID2:
 217                 {
 218                                 int i;
 219 @@ -2993,12 +3071,13 @@ static long vc_sm_ioctl(struct file *fil
 220                                 for (i = 0; i < ioparam.op_count; i++) {
 221                                         const struct vmcs_sm_ioctl_clean_invalid_block * const op = block + i;
 222
 223 -                                       ret = clean_invalid_mem_2d((void __user*) op->start_address,
 224 -                                                       op->block_count, op->block_size,
 225 -                                                       op->inter_block_stride, op->invalidate_mode);
 226                                         if (op->invalidate_mode == VCSM_CACHE_OP_NOP)
 227                                                 continue;
 228
 229 +                                       ret = clean_invalid_contiguous_mem_2d(
 230 +                                                       (void __user*) op->start_address, op->block_count,
 231 +                                                       op->block_size, op->inter_block_stride,
 232 +                                                       op->invalidate_mode);
 233                                         if (ret)
 234                                                 break;
 235                                 }