libsparse/sparse_read.c

   1 /*
   2  * Copyright (C) 2012 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #define _GNU_SOURCE
  18 #define _FILE_OFFSET_BITS 64
  19 #define _LARGEFILE64_SOURCE 1
  20
  21 #include <fcntl.h>
  22 #include <stdarg.h>
  23 #include <stdbool.h>
  24 #include <stdint.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <string.h>
  28 #include <unistd.h>
  29
  30 #include <sparse/sparse.h>
  31
  32 #include "defs.h"
  33 #include "output_file.h"
  34 #include "sparse_crc32.h"
  35 #include "sparse_file.h"
  36 #include "sparse_format.h"
  37
  38 #define SPARSE_HEADER_MAJOR_VER 1
  39 #define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
  40 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
  41
  42 #define COPY_BUF_SIZE (1024U*1024U)
  43 static char *copybuf;
  44
  45 #define min(a, b) \
  46         ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
  47
  48 static void verbose_error(bool verbose, int err, const char *fmt, ...)
  49 {
  50         char *s = "";
  51         char *at = "";
  52         if (fmt) {
  53                 va_list argp;
  54                 int size;
  55
  56                 va_start(argp, fmt);
  57                 size = vsnprintf(NULL, 0, fmt, argp);
  58                 va_end(argp);
  59
  60                 if (size < 0) {
  61                         return;
  62                 }
  63
  64                 at = malloc(size + 1);
  65                 if (at == NULL) {
  66                         return;
  67                 }
  68
  69                 va_start(argp, fmt);
  70                 vsnprintf(at, size, fmt, argp);
  71                 va_end(argp);
  72                 at[size] = 0;
  73                 s = " at ";
  74         }
  75         if (verbose) {
  76                 if (err == -EOVERFLOW) {
  77                         sparse_print_verbose("EOF while reading file%s%s\n", s, at);
  78                 } else if (err == -EINVAL) {
  79                         sparse_print_verbose("Invalid sparse file format%s%s\n", s, at);
  80                 } else if (err == -ENOMEM) {
  81                         sparse_print_verbose("Failed allocation while reading file%s%s\n",
  82                                         s, at);
  83                 } else {
  84                         sparse_print_verbose("Unknown error %d%s%s\n", err, s, at);
  85                 }
  86         }
  87         if (fmt) {
  88                 free(at);
  89         }
  90 }
  91
  92 static int process_raw_chunk(struct sparse_file *s, unsigned int chunk_size,
  93                 int fd, int64_t offset, unsigned int blocks, unsigned int block,
  94                 uint32_t *crc32)
  95 {
  96         int ret;
  97         int chunk;
  98         unsigned int len = blocks * s->block_size;
  99
 100         if (chunk_size % s->block_size != 0) {
 101                 return -EINVAL;
 102         }
 103
 104         if (chunk_size / s->block_size != blocks) {
 105                 return -EINVAL;
 106         }
 107
 108         ret = sparse_file_add_fd(s, fd, offset, len, block);
 109         if (ret < 0) {
 110                 return ret;
 111         }
 112
 113         if (crc32) {
 114                 while (len) {
 115                         chunk = min(len, COPY_BUF_SIZE);
 116                         ret = read_all(fd, copybuf, chunk);
 117                         if (ret < 0) {
 118                                 return ret;
 119                         }
 120                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 121                         len -= chunk;
 122                 }
 123         } else {
 124                 lseek(fd, len, SEEK_CUR);
 125         }
 126
 127         return 0;
 128 }
 129
 130 static int process_fill_chunk(struct sparse_file *s, unsigned int chunk_size,
 131                 int fd, unsigned int blocks, unsigned int block, uint32_t *crc32)
 132 {
 133         int ret;
 134         int chunk;
 135         int64_t len = (int64_t)blocks * s->block_size;
 136         uint32_t fill_val;
 137         uint32_t *fillbuf;
 138         unsigned int i;
 139
 140         if (chunk_size != sizeof(fill_val)) {
 141                 return -EINVAL;
 142         }
 143
 144         ret = read_all(fd, &fill_val, sizeof(fill_val));
 145         if (ret < 0) {
 146                 return ret;
 147         }
 148
 149         ret = sparse_file_add_fill(s, fill_val, len, block);
 150         if (ret < 0) {
 151                 return ret;
 152         }
 153
 154         if (crc32) {
 155                 /* Fill copy_buf with the fill value */
 156                 fillbuf = (uint32_t *)copybuf;
 157                 for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) {
 158                         fillbuf[i] = fill_val;
 159                 }
 160
 161                 while (len) {
 162                         chunk = min(len, COPY_BUF_SIZE);
 163                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 164                         len -= chunk;
 165                 }
 166         }
 167
 168         return 0;
 169 }
 170
 171 static int process_skip_chunk(struct sparse_file *s, unsigned int chunk_size,
 172                 int fd __unused, unsigned int blocks,
 173                 unsigned int block __unused, uint32_t *crc32)
 174 {
 175         if (chunk_size != 0) {
 176                 return -EINVAL;
 177         }
 178
 179         if (crc32) {
 180                 int64_t len = (int64_t)blocks * s->block_size;
 181                 memset(copybuf, 0, COPY_BUF_SIZE);
 182
 183                 while (len) {
 184                         int chunk = min(len, COPY_BUF_SIZE);
 185                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 186                         len -= chunk;
 187                 }
 188         }
 189
 190         return 0;
 191 }
 192
 193 static int process_crc32_chunk(int fd, unsigned int chunk_size, uint32_t crc32)
 194 {
 195         uint32_t file_crc32;
 196         int ret;
 197
 198         if (chunk_size != sizeof(file_crc32)) {
 199                 return -EINVAL;
 200         }
 201
 202         ret = read_all(fd, &file_crc32, sizeof(file_crc32));
 203         if (ret < 0) {
 204                 return ret;
 205         }
 206
 207         if (file_crc32 != crc32) {
 208                 return -EINVAL;
 209         }
 210
 211         return 0;
 212 }
 213
 214 static int process_chunk(struct sparse_file *s, int fd, off_t offset,
 215                 unsigned int chunk_hdr_sz, chunk_header_t *chunk_header,
 216                 unsigned int cur_block, uint32_t *crc_ptr)
 217 {
 218         int ret;
 219         unsigned int chunk_data_size;
 220
 221         chunk_data_size = chunk_header->total_sz - chunk_hdr_sz;
 222
 223         switch (chunk_header->chunk_type) {
 224                 case CHUNK_TYPE_RAW:
 225                         ret = process_raw_chunk(s, chunk_data_size, fd, offset,
 226                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 227                         if (ret < 0) {
 228                                 verbose_error(s->verbose, ret, "data block at %lld", offset);
 229                                 return ret;
 230                         }
 231                         return chunk_header->chunk_sz;
 232                 case CHUNK_TYPE_FILL:
 233                         ret = process_fill_chunk(s, chunk_data_size, fd,
 234                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 235                         if (ret < 0) {
 236                                 verbose_error(s->verbose, ret, "fill block at %lld", offset);
 237                                 return ret;
 238                         }
 239                         return chunk_header->chunk_sz;
 240                 case CHUNK_TYPE_DONT_CARE:
 241                         ret = process_skip_chunk(s, chunk_data_size, fd,
 242                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 243                         if (chunk_data_size != 0) {
 244                                 if (ret < 0) {
 245                                         verbose_error(s->verbose, ret, "skip block at %lld", offset);
 246                                         return ret;
 247                                 }
 248                         }
 249                         return chunk_header->chunk_sz;
 250                 case CHUNK_TYPE_CRC32:
 251                         ret = process_crc32_chunk(fd, chunk_data_size, *crc_ptr);
 252                         if (ret < 0) {
 253                                 verbose_error(s->verbose, -EINVAL, "crc block at %lld",
 254                                                 offset);
 255                                 return ret;
 256                         }
 257                         return 0;
 258                 default:
 259                         verbose_error(s->verbose, -EINVAL, "unknown block %04X at %lld",
 260                                         chunk_header->chunk_type, offset);
 261         }
 262
 263         return 0;
 264 }
 265
 266 static int sparse_file_read_sparse(struct sparse_file *s, int fd, bool crc)
 267 {
 268         int ret;
 269         unsigned int i;
 270         sparse_header_t sparse_header;
 271         chunk_header_t chunk_header;
 272         uint32_t crc32 = 0;
 273         uint32_t *crc_ptr = 0;
 274         unsigned int cur_block = 0;
 275         off_t offset;
 276
 277         if (!copybuf) {
 278                 copybuf = malloc(COPY_BUF_SIZE);
 279         }
 280
 281         if (!copybuf) {
 282                 return -ENOMEM;
 283         }
 284
 285         if (crc) {
 286                 crc_ptr = &crc32;
 287         }
 288
 289         ret = read_all(fd, &sparse_header, sizeof(sparse_header));
 290         if (ret < 0) {
 291                 return ret;
 292         }
 293
 294         if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
 295                 return -EINVAL;
 296         }
 297
 298         if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
 299                 return -EINVAL;
 300         }
 301
 302         if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
 303                 return -EINVAL;
 304         }
 305
 306         if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) {
 307                 return -EINVAL;
 308         }
 309
 310         if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) {
 311                 /* Skip the remaining bytes in a header that is longer than
 312                  * we expected.
 313                  */
 314                 lseek(fd, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR);
 315         }
 316
 317         for (i = 0; i < sparse_header.total_chunks; i++) {
 318                 ret = read_all(fd, &chunk_header, sizeof(chunk_header));
 319                 if (ret < 0) {
 320                         return ret;
 321                 }
 322
 323                 if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) {
 324                         /* Skip the remaining bytes in a header that is longer than
 325                          * we expected.
 326                          */
 327                         lseek(fd, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR);
 328                 }
 329
 330                 offset = lseek(fd, 0, SEEK_CUR);
 331
 332                 ret = process_chunk(s, fd, offset, sparse_header.chunk_hdr_sz, &chunk_header,
 333                                 cur_block, crc_ptr);
 334                 if (ret < 0) {
 335                         return ret;
 336                 }
 337
 338                 cur_block += ret;
 339         }
 340
 341         if (sparse_header.total_blks != cur_block) {
 342                 return -EINVAL;
 343         }
 344
 345         return 0;
 346 }
 347
 348 static int sparse_file_read_normal(struct sparse_file *s, int fd)
 349 {
 350         int ret;
 351         uint32_t *buf = malloc(s->block_size);
 352         unsigned int block = 0;
 353         int64_t remain = s->len;
 354         int64_t offset = 0;
 355         unsigned int to_read;
 356         unsigned int i;
 357         bool sparse_block;
 358
 359         if (!buf) {
 360                 return -ENOMEM;
 361         }
 362
 363         while (remain > 0) {
 364                 to_read = min(remain, s->block_size);
 365                 ret = read_all(fd, buf, to_read);
 366                 if (ret < 0) {
 367                         error("failed to read sparse file");
 368                         return ret;
 369                 }
 370
 371                 if (to_read == s->block_size) {
 372                         sparse_block = true;
 373                         for (i = 1; i < s->block_size / sizeof(uint32_t); i++) {
 374                                 if (buf[0] != buf[i]) {
 375                                         sparse_block = false;
 376                                         break;
 377                                 }
 378                         }
 379                 } else {
 380                         sparse_block = false;
 381                 }
 382
 383                 if (sparse_block) {
 384                         /* TODO: add flag to use skip instead of fill for buf[0] == 0 */
 385                         sparse_file_add_fill(s, buf[0], to_read, block);
 386                 } else {
 387                         sparse_file_add_fd(s, fd, offset, to_read, block);
 388                 }
 389
 390                 remain -= to_read;
 391                 offset += to_read;
 392                 block++;
 393         }
 394
 395         return 0;
 396 }
 397
 398 int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc)
 399 {
 400         if (crc && !sparse) {
 401                 return -EINVAL;
 402         }
 403
 404         if (sparse) {
 405                 return sparse_file_read_sparse(s, fd, crc);
 406         } else {
 407                 return sparse_file_read_normal(s, fd);
 408         }
 409 }
 410
 411 struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc)
 412 {
 413         int ret;
 414         sparse_header_t sparse_header;
 415         int64_t len;
 416         struct sparse_file *s;
 417
 418         ret = read_all(fd, &sparse_header, sizeof(sparse_header));
 419         if (ret < 0) {
 420                 verbose_error(verbose, ret, "header");
 421                 return NULL;
 422         }
 423
 424         if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
 425                 verbose_error(verbose, -EINVAL, "header magic");
 426                 return NULL;
 427         }
 428
 429         if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
 430                 verbose_error(verbose, -EINVAL, "header major version");
 431                 return NULL;
 432         }
 433
 434         if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
 435                 return NULL;
 436         }
 437
 438         if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) {
 439                 return NULL;
 440         }
 441
 442         len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz;
 443         s = sparse_file_new(sparse_header.blk_sz, len);
 444         if (!s) {
 445                 verbose_error(verbose, -EINVAL, NULL);
 446                 return NULL;
 447         }
 448
 449         ret = lseek(fd, 0, SEEK_SET);
 450         if (ret < 0) {
 451                 verbose_error(verbose, ret, "seeking");
 452                 sparse_file_destroy(s);
 453                 return NULL;
 454         }
 455
 456         s->verbose = verbose;
 457
 458         ret = sparse_file_read(s, fd, true, crc);
 459         if (ret < 0) {
 460                 sparse_file_destroy(s);
 461                 return NULL;
 462         }
 463
 464         return s;
 465 }
 466
 467 struct sparse_file *sparse_file_import_auto(int fd, bool crc, bool verbose)
 468 {
 469         struct sparse_file *s;
 470         int64_t len;
 471         int ret;
 472
 473         s = sparse_file_import(fd, verbose, crc);
 474         if (s) {
 475                 return s;
 476         }
 477
 478         len = lseek(fd, 0, SEEK_END);
 479         if (len < 0) {
 480                 return NULL;
 481         }
 482
 483         lseek(fd, 0, SEEK_SET);
 484
 485         s = sparse_file_new(4096, len);
 486         if (!s) {
 487                 return NULL;
 488         }
 489
 490         ret = sparse_file_read_normal(s, fd);
 491         if (ret < 0) {
 492                 sparse_file_destroy(s);
 493                 return NULL;
 494         }
 495
 496         return s;
 497 }