libsparse/sparse_read.c

   1 /*
   2  * Copyright (C) 2012 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #define _GNU_SOURCE
  18 #define _FILE_OFFSET_BITS 64
  19 #define _LARGEFILE64_SOURCE 1
  20
  21 #include <fcntl.h>
  22 #include <stdarg.h>
  23 #include <stdbool.h>
  24 #include <stdint.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <string.h>
  28 #include <unistd.h>
  29
  30 #include <sparse/sparse.h>
  31
  32 #include "defs.h"
  33 #include "output_file.h"
  34 #include "sparse_crc32.h"
  35 #include "sparse_file.h"
  36 #include "sparse_format.h"
  37
  38 #if defined(__APPLE__) && defined(__MACH__)
  39 #define lseek64 lseek
  40 #define off64_t off_t
  41 #endif
  42
  43 #define SPARSE_HEADER_MAJOR_VER 1
  44 #define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
  45 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
  46
  47 #define COPY_BUF_SIZE (1024U*1024U)
  48 static char *copybuf;
  49
  50 #define min(a, b) \
  51         ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
  52
  53 static void verbose_error(bool verbose, int err, const char *fmt, ...)
  54 {
  55         char *s = "";
  56         char *at = "";
  57         if (fmt) {
  58                 va_list argp;
  59                 int size;
  60
  61                 va_start(argp, fmt);
  62                 size = vsnprintf(NULL, 0, fmt, argp);
  63                 va_end(argp);
  64
  65                 if (size < 0) {
  66                         return;
  67                 }
  68
  69                 at = malloc(size + 1);
  70                 if (at == NULL) {
  71                         return;
  72                 }
  73
  74                 va_start(argp, fmt);
  75                 vsnprintf(at, size, fmt, argp);
  76                 va_end(argp);
  77                 at[size] = 0;
  78                 s = " at ";
  79         }
  80         if (verbose) {
  81 #ifndef USE_MINGW
  82                 if (err == -EOVERFLOW) {
  83                         sparse_print_verbose("EOF while reading file%s%s\n", s, at);
  84                 } else
  85 #endif
  86                 if (err == -EINVAL) {
  87                         sparse_print_verbose("Invalid sparse file format%s%s\n", s, at);
  88                 } else if (err == -ENOMEM) {
  89                         sparse_print_verbose("Failed allocation while reading file%s%s\n",
  90                                         s, at);
  91                 } else {
  92                         sparse_print_verbose("Unknown error %d%s%s\n", err, s, at);
  93                 }
  94         }
  95         if (fmt) {
  96                 free(at);
  97         }
  98 }
  99
 100 static int process_raw_chunk(struct sparse_file *s, unsigned int chunk_size,
 101                 int fd, int64_t offset, unsigned int blocks, unsigned int block,
 102                 uint32_t *crc32)
 103 {
 104         int ret;
 105         int chunk;
 106         unsigned int len = blocks * s->block_size;
 107
 108         if (chunk_size % s->block_size != 0) {
 109                 return -EINVAL;
 110         }
 111
 112         if (chunk_size / s->block_size != blocks) {
 113                 return -EINVAL;
 114         }
 115
 116         ret = sparse_file_add_fd(s, fd, offset, len, block);
 117         if (ret < 0) {
 118                 return ret;
 119         }
 120
 121         if (crc32) {
 122                 while (len) {
 123                         chunk = min(len, COPY_BUF_SIZE);
 124                         ret = read_all(fd, copybuf, chunk);
 125                         if (ret < 0) {
 126                                 return ret;
 127                         }
 128                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 129                         len -= chunk;
 130                 }
 131         } else {
 132                 lseek64(fd, len, SEEK_CUR);
 133         }
 134
 135         return 0;
 136 }
 137
 138 static int process_fill_chunk(struct sparse_file *s, unsigned int chunk_size,
 139                 int fd, unsigned int blocks, unsigned int block, uint32_t *crc32)
 140 {
 141         int ret;
 142         int chunk;
 143         int64_t len = (int64_t)blocks * s->block_size;
 144         uint32_t fill_val;
 145         uint32_t *fillbuf;
 146         unsigned int i;
 147
 148         if (chunk_size != sizeof(fill_val)) {
 149                 return -EINVAL;
 150         }
 151
 152         ret = read_all(fd, &fill_val, sizeof(fill_val));
 153         if (ret < 0) {
 154                 return ret;
 155         }
 156
 157         ret = sparse_file_add_fill(s, fill_val, len, block);
 158         if (ret < 0) {
 159                 return ret;
 160         }
 161
 162         if (crc32) {
 163                 /* Fill copy_buf with the fill value */
 164                 fillbuf = (uint32_t *)copybuf;
 165                 for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) {
 166                         fillbuf[i] = fill_val;
 167                 }
 168
 169                 while (len) {
 170                         chunk = min(len, COPY_BUF_SIZE);
 171                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 172                         len -= chunk;
 173                 }
 174         }
 175
 176         return 0;
 177 }
 178
 179 static int process_skip_chunk(struct sparse_file *s, unsigned int chunk_size,
 180                 int fd __unused, unsigned int blocks,
 181                 unsigned int block __unused, uint32_t *crc32)
 182 {
 183         if (chunk_size != 0) {
 184                 return -EINVAL;
 185         }
 186
 187         if (crc32) {
 188                 int64_t len = (int64_t)blocks * s->block_size;
 189                 memset(copybuf, 0, COPY_BUF_SIZE);
 190
 191                 while (len) {
 192                         int chunk = min(len, COPY_BUF_SIZE);
 193                         *crc32 = sparse_crc32(*crc32, copybuf, chunk);
 194                         len -= chunk;
 195                 }
 196         }
 197
 198         return 0;
 199 }
 200
 201 static int process_crc32_chunk(int fd, unsigned int chunk_size, uint32_t crc32)
 202 {
 203         uint32_t file_crc32;
 204         int ret;
 205
 206         if (chunk_size != sizeof(file_crc32)) {
 207                 return -EINVAL;
 208         }
 209
 210         ret = read_all(fd, &file_crc32, sizeof(file_crc32));
 211         if (ret < 0) {
 212                 return ret;
 213         }
 214
 215         if (file_crc32 != crc32) {
 216                 return -EINVAL;
 217         }
 218
 219         return 0;
 220 }
 221
 222 static int process_chunk(struct sparse_file *s, int fd, off64_t offset,
 223                 unsigned int chunk_hdr_sz, chunk_header_t *chunk_header,
 224                 unsigned int cur_block, uint32_t *crc_ptr)
 225 {
 226         int ret;
 227         unsigned int chunk_data_size;
 228
 229         chunk_data_size = chunk_header->total_sz - chunk_hdr_sz;
 230
 231         switch (chunk_header->chunk_type) {
 232                 case CHUNK_TYPE_RAW:
 233                         ret = process_raw_chunk(s, chunk_data_size, fd, offset,
 234                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 235                         if (ret < 0) {
 236                                 verbose_error(s->verbose, ret, "data block at %lld", offset);
 237                                 return ret;
 238                         }
 239                         return chunk_header->chunk_sz;
 240                 case CHUNK_TYPE_FILL:
 241                         ret = process_fill_chunk(s, chunk_data_size, fd,
 242                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 243                         if (ret < 0) {
 244                                 verbose_error(s->verbose, ret, "fill block at %lld", offset);
 245                                 return ret;
 246                         }
 247                         return chunk_header->chunk_sz;
 248                 case CHUNK_TYPE_DONT_CARE:
 249                         ret = process_skip_chunk(s, chunk_data_size, fd,
 250                                         chunk_header->chunk_sz, cur_block, crc_ptr);
 251                         if (chunk_data_size != 0) {
 252                                 if (ret < 0) {
 253                                         verbose_error(s->verbose, ret, "skip block at %lld", offset);
 254                                         return ret;
 255                                 }
 256                         }
 257                         return chunk_header->chunk_sz;
 258                 case CHUNK_TYPE_CRC32:
 259                         ret = process_crc32_chunk(fd, chunk_data_size, *crc_ptr);
 260                         if (ret < 0) {
 261                                 verbose_error(s->verbose, -EINVAL, "crc block at %lld",
 262                                                 offset);
 263                                 return ret;
 264                         }
 265                         return 0;
 266                 default:
 267                         verbose_error(s->verbose, -EINVAL, "unknown block %04X at %lld",
 268                                         chunk_header->chunk_type, offset);
 269         }
 270
 271         return 0;
 272 }
 273
 274 static int sparse_file_read_sparse(struct sparse_file *s, int fd, bool crc)
 275 {
 276         int ret;
 277         unsigned int i;
 278         sparse_header_t sparse_header;
 279         chunk_header_t chunk_header;
 280         uint32_t crc32 = 0;
 281         uint32_t *crc_ptr = 0;
 282         unsigned int cur_block = 0;
 283         off64_t offset;
 284
 285         if (!copybuf) {
 286                 copybuf = malloc(COPY_BUF_SIZE);
 287         }
 288
 289         if (!copybuf) {
 290                 return -ENOMEM;
 291         }
 292
 293         if (crc) {
 294                 crc_ptr = &crc32;
 295         }
 296
 297         ret = read_all(fd, &sparse_header, sizeof(sparse_header));
 298         if (ret < 0) {
 299                 return ret;
 300         }
 301
 302         if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
 303                 return -EINVAL;
 304         }
 305
 306         if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
 307                 return -EINVAL;
 308         }
 309
 310         if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
 311                 return -EINVAL;
 312         }
 313
 314         if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) {
 315                 return -EINVAL;
 316         }
 317
 318         if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) {
 319                 /* Skip the remaining bytes in a header that is longer than
 320                  * we expected.
 321                  */
 322                 lseek64(fd, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR);
 323         }
 324
 325         for (i = 0; i < sparse_header.total_chunks; i++) {
 326                 ret = read_all(fd, &chunk_header, sizeof(chunk_header));
 327                 if (ret < 0) {
 328                         return ret;
 329                 }
 330
 331                 if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) {
 332                         /* Skip the remaining bytes in a header that is longer than
 333                          * we expected.
 334                          */
 335                         lseek64(fd, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR);
 336                 }
 337
 338                 offset = lseek64(fd, 0, SEEK_CUR);
 339
 340                 ret = process_chunk(s, fd, offset, sparse_header.chunk_hdr_sz, &chunk_header,
 341                                 cur_block, crc_ptr);
 342                 if (ret < 0) {
 343                         return ret;
 344                 }
 345
 346                 cur_block += ret;
 347         }
 348
 349         if (sparse_header.total_blks != cur_block) {
 350                 return -EINVAL;
 351         }
 352
 353         return 0;
 354 }
 355
 356 static int sparse_file_read_normal(struct sparse_file *s, int fd)
 357 {
 358         int ret;
 359         uint32_t *buf = malloc(s->block_size);
 360         unsigned int block = 0;
 361         int64_t remain = s->len;
 362         int64_t offset = 0;
 363         unsigned int to_read;
 364         unsigned int i;
 365         bool sparse_block;
 366
 367         if (!buf) {
 368                 return -ENOMEM;
 369         }
 370
 371         while (remain > 0) {
 372                 to_read = min(remain, s->block_size);
 373                 ret = read_all(fd, buf, to_read);
 374                 if (ret < 0) {
 375                         error("failed to read sparse file");
 376                         return ret;
 377                 }
 378
 379                 if (to_read == s->block_size) {
 380                         sparse_block = true;
 381                         for (i = 1; i < s->block_size / sizeof(uint32_t); i++) {
 382                                 if (buf[0] != buf[i]) {
 383                                         sparse_block = false;
 384                                         break;
 385                                 }
 386                         }
 387                 } else {
 388                         sparse_block = false;
 389                 }
 390
 391                 if (sparse_block) {
 392                         /* TODO: add flag to use skip instead of fill for buf[0] == 0 */
 393                         sparse_file_add_fill(s, buf[0], to_read, block);
 394                 } else {
 395                         sparse_file_add_fd(s, fd, offset, to_read, block);
 396                 }
 397
 398                 remain -= to_read;
 399                 offset += to_read;
 400                 block++;
 401         }
 402
 403         return 0;
 404 }
 405
 406 int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc)
 407 {
 408         if (crc && !sparse) {
 409                 return -EINVAL;
 410         }
 411
 412         if (sparse) {
 413                 return sparse_file_read_sparse(s, fd, crc);
 414         } else {
 415                 return sparse_file_read_normal(s, fd);
 416         }
 417 }
 418
 419 struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc)
 420 {
 421         int ret;
 422         sparse_header_t sparse_header;
 423         int64_t len;
 424         struct sparse_file *s;
 425
 426         ret = read_all(fd, &sparse_header, sizeof(sparse_header));
 427         if (ret < 0) {
 428                 verbose_error(verbose, ret, "header");
 429                 return NULL;
 430         }
 431
 432         if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
 433                 verbose_error(verbose, -EINVAL, "header magic");
 434                 return NULL;
 435         }
 436
 437         if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
 438                 verbose_error(verbose, -EINVAL, "header major version");
 439                 return NULL;
 440         }
 441
 442         if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
 443                 return NULL;
 444         }
 445
 446         if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) {
 447                 return NULL;
 448         }
 449
 450         len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz;
 451         s = sparse_file_new(sparse_header.blk_sz, len);
 452         if (!s) {
 453                 verbose_error(verbose, -EINVAL, NULL);
 454                 return NULL;
 455         }
 456
 457         ret = lseek64(fd, 0, SEEK_SET);
 458         if (ret < 0) {
 459                 verbose_error(verbose, ret, "seeking");
 460                 sparse_file_destroy(s);
 461                 return NULL;
 462         }
 463
 464         s->verbose = verbose;
 465
 466         ret = sparse_file_read(s, fd, true, crc);
 467         if (ret < 0) {
 468                 sparse_file_destroy(s);
 469                 return NULL;
 470         }
 471
 472         return s;
 473 }
 474
 475 struct sparse_file *sparse_file_import_auto(int fd, bool crc, bool verbose)
 476 {
 477         struct sparse_file *s;
 478         int64_t len;
 479         int ret;
 480
 481         s = sparse_file_import(fd, verbose, crc);
 482         if (s) {
 483                 return s;
 484         }
 485
 486         len = lseek64(fd, 0, SEEK_END);
 487         if (len < 0) {
 488                 return NULL;
 489         }
 490
 491         lseek64(fd, 0, SEEK_SET);
 492
 493         s = sparse_file_new(4096, len);
 494         if (!s) {
 495                 return NULL;
 496         }
 497
 498         ret = sparse_file_read_normal(s, fd);
 499         if (ret < 0) {
 500                 sparse_file_destroy(s);
 501                 return NULL;
 502         }
 503
 504         return s;
 505 }