libsparse/output_file.c

   1 /*
   2  * Copyright (C) 2010 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #define _FILE_OFFSET_BITS 64
  18 #define _LARGEFILE64_SOURCE 1
  19
  20 #include <fcntl.h>
  21 #include <inttypes.h>
  22 #include <limits.h>
  23 #include <stdbool.h>
  24 #include <stddef.h>
  25 #include <stdlib.h>
  26 #include <string.h>
  27 #include <sys/stat.h>
  28 #include <sys/types.h>
  29 #include <unistd.h>
  30 #include <zlib.h>
  31
  32 #include "defs.h"
  33 #include "output_file.h"
  34 #include "sparse_crc32.h"
  35 #include "sparse_format.h"
  36
  37 #ifndef USE_MINGW
  38 #include <sys/mman.h>
  39 #define O_BINARY 0
  40 #else
  41 #define ftruncate64 ftruncate
  42 #endif
  43
  44 #if defined(__APPLE__) && defined(__MACH__)
  45 #define lseek64 lseek
  46 #define ftruncate64 ftruncate
  47 #define mmap64 mmap
  48 #define off64_t off_t
  49 #endif
  50
  51 #define min(a, b) \
  52         ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
  53
  54 #define SPARSE_HEADER_MAJOR_VER 1
  55 #define SPARSE_HEADER_MINOR_VER 0
  56 #define SPARSE_HEADER_LEN       (sizeof(sparse_header_t))
  57 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
  58
  59 #define container_of(inner, outer_t, elem) \
  60         ((outer_t *)((char *)inner - offsetof(outer_t, elem)))
  61
  62 struct output_file_ops {
  63         int (*open)(struct output_file *, int fd);
  64         int (*skip)(struct output_file *, int64_t);
  65         int (*pad)(struct output_file *, int64_t);
  66         int (*write)(struct output_file *, void *, int);
  67         void (*close)(struct output_file *);
  68 };
  69
  70 struct sparse_file_ops {
  71         int (*write_data_chunk)(struct output_file *out, unsigned int len,
  72                         void *data);
  73         int (*write_fill_chunk)(struct output_file *out, unsigned int len,
  74                         uint32_t fill_val);
  75         int (*write_skip_chunk)(struct output_file *out, int64_t len);
  76         int (*write_end_chunk)(struct output_file *out);
  77 };
  78
  79 struct output_file {
  80         int64_t cur_out_ptr;
  81         unsigned int chunk_cnt;
  82         uint32_t crc32;
  83         struct output_file_ops *ops;
  84         struct sparse_file_ops *sparse_ops;
  85         int use_crc;
  86         unsigned int block_size;
  87         int64_t len;
  88         char *zero_buf;
  89         uint32_t *fill_buf;
  90         char *buf;
  91 };
  92
  93 struct output_file_gz {
  94         struct output_file out;
  95         gzFile gz_fd;
  96 };
  97
  98 #define to_output_file_gz(_o) \
  99         container_of((_o), struct output_file_gz, out)
 100
 101 struct output_file_normal {
 102         struct output_file out;
 103         int fd;
 104 };
 105
 106 #define to_output_file_normal(_o) \
 107         container_of((_o), struct output_file_normal, out)
 108
 109 struct output_file_callback {
 110         struct output_file out;
 111         void *priv;
 112         int (*write)(void *priv, const void *buf, int len);
 113 };
 114
 115 #define to_output_file_callback(_o) \
 116         container_of((_o), struct output_file_callback, out)
 117
 118 static int file_open(struct output_file *out, int fd)
 119 {
 120         struct output_file_normal *outn = to_output_file_normal(out);
 121
 122         outn->fd = fd;
 123         return 0;
 124 }
 125
 126 static int file_skip(struct output_file *out, int64_t cnt)
 127 {
 128         off64_t ret;
 129         struct output_file_normal *outn = to_output_file_normal(out);
 130
 131         ret = lseek64(outn->fd, cnt, SEEK_CUR);
 132         if (ret < 0) {
 133                 error_errno("lseek64");
 134                 return -1;
 135         }
 136         return 0;
 137 }
 138
 139 static int file_pad(struct output_file *out, int64_t len)
 140 {
 141         int ret;
 142         struct output_file_normal *outn = to_output_file_normal(out);
 143
 144         ret = ftruncate64(outn->fd, len);
 145         if (ret < 0) {
 146                 return -errno;
 147         }
 148
 149         return 0;
 150 }
 151
 152 static int file_write(struct output_file *out, void *data, int len)
 153 {
 154         int ret;
 155         struct output_file_normal *outn = to_output_file_normal(out);
 156
 157         ret = write(outn->fd, data, len);
 158         if (ret < 0) {
 159                 error_errno("write");
 160                 return -1;
 161         } else if (ret < len) {
 162                 error("incomplete write");
 163                 return -1;
 164         }
 165
 166         return 0;
 167 }
 168
 169 static void file_close(struct output_file *out)
 170 {
 171         struct output_file_normal *outn = to_output_file_normal(out);
 172
 173         free(outn);
 174 }
 175
 176 static struct output_file_ops file_ops = {
 177         .open = file_open,
 178         .skip = file_skip,
 179         .pad = file_pad,
 180         .write = file_write,
 181         .close = file_close,
 182 };
 183
 184 static int gz_file_open(struct output_file *out, int fd)
 185 {
 186         struct output_file_gz *outgz = to_output_file_gz(out);
 187
 188         outgz->gz_fd = gzdopen(fd, "wb9");
 189         if (!outgz->gz_fd) {
 190                 error_errno("gzopen");
 191                 return -errno;
 192         }
 193
 194         return 0;
 195 }
 196
 197
 198 static int gz_file_skip(struct output_file *out, int64_t cnt)
 199 {
 200         off64_t ret;
 201         struct output_file_gz *outgz = to_output_file_gz(out);
 202
 203         ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
 204         if (ret < 0) {
 205                 error_errno("gzseek");
 206                 return -1;
 207         }
 208         return 0;
 209 }
 210
 211 static int gz_file_pad(struct output_file *out, int64_t len)
 212 {
 213         off64_t ret;
 214         struct output_file_gz *outgz = to_output_file_gz(out);
 215
 216         ret = gztell(outgz->gz_fd);
 217         if (ret < 0) {
 218                 return -1;
 219         }
 220
 221         if (ret >= len) {
 222                 return 0;
 223         }
 224
 225         ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
 226         if (ret < 0) {
 227                 return -1;
 228         }
 229
 230         gzwrite(outgz->gz_fd, "", 1);
 231
 232         return 0;
 233 }
 234
 235 static int gz_file_write(struct output_file *out, void *data, int len)
 236 {
 237         int ret;
 238         struct output_file_gz *outgz = to_output_file_gz(out);
 239
 240         ret = gzwrite(outgz->gz_fd, data, len);
 241         if (ret < 0) {
 242                 error_errno("gzwrite");
 243                 return -1;
 244         } else if (ret < len) {
 245                 error("incomplete gzwrite");
 246                 return -1;
 247         }
 248
 249         return 0;
 250 }
 251
 252 static void gz_file_close(struct output_file *out)
 253 {
 254         struct output_file_gz *outgz = to_output_file_gz(out);
 255
 256         gzclose(outgz->gz_fd);
 257         free(outgz);
 258 }
 259
 260 static struct output_file_ops gz_file_ops = {
 261         .open = gz_file_open,
 262         .skip = gz_file_skip,
 263         .pad = gz_file_pad,
 264         .write = gz_file_write,
 265         .close = gz_file_close,
 266 };
 267
 268 static int callback_file_open(struct output_file *out __unused, int fd __unused)
 269 {
 270         return 0;
 271 }
 272
 273 static int callback_file_skip(struct output_file *out, int64_t off)
 274 {
 275         struct output_file_callback *outc = to_output_file_callback(out);
 276         int to_write;
 277         int ret;
 278
 279         while (off > 0) {
 280                 to_write = min(off, (int64_t)INT_MAX);
 281                 ret = outc->write(outc->priv, NULL, to_write);
 282                 if (ret < 0) {
 283                         return ret;
 284                 }
 285                 off -= to_write;
 286         }
 287
 288         return 0;
 289 }
 290
 291 static int callback_file_pad(struct output_file *out __unused, int64_t len __unused)
 292 {
 293         return -1;
 294 }
 295
 296 static int callback_file_write(struct output_file *out, void *data, int len)
 297 {
 298         struct output_file_callback *outc = to_output_file_callback(out);
 299
 300         return outc->write(outc->priv, data, len);
 301 }
 302
 303 static void callback_file_close(struct output_file *out)
 304 {
 305         struct output_file_callback *outc = to_output_file_callback(out);
 306
 307         free(outc);
 308 }
 309
 310 static struct output_file_ops callback_file_ops = {
 311         .open = callback_file_open,
 312         .skip = callback_file_skip,
 313         .pad = callback_file_pad,
 314         .write = callback_file_write,
 315         .close = callback_file_close,
 316 };
 317
 318 int read_all(int fd, void *buf, size_t len)
 319 {
 320         size_t total = 0;
 321         int ret;
 322         char *ptr = buf;
 323
 324         while (total < len) {
 325                 ret = read(fd, ptr, len - total);
 326
 327                 if (ret < 0)
 328                         return -errno;
 329
 330                 if (ret == 0)
 331                         return -EINVAL;
 332
 333                 ptr += ret;
 334                 total += ret;
 335         }
 336
 337         return 0;
 338 }
 339
 340 static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
 341 {
 342         chunk_header_t chunk_header;
 343         int ret;
 344
 345         if (skip_len % out->block_size) {
 346                 error("don't care size %"PRIi64" is not a multiple of the block size %u",
 347                                 skip_len, out->block_size);
 348                 return -1;
 349         }
 350
 351         /* We are skipping data, so emit a don't care chunk. */
 352         chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
 353         chunk_header.reserved1 = 0;
 354         chunk_header.chunk_sz = skip_len / out->block_size;
 355         chunk_header.total_sz = CHUNK_HEADER_LEN;
 356         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 357         if (ret < 0)
 358                 return -1;
 359
 360         out->cur_out_ptr += skip_len;
 361         out->chunk_cnt++;
 362
 363         return 0;
 364 }
 365
 366 static int write_sparse_fill_chunk(struct output_file *out, unsigned int len,
 367                 uint32_t fill_val)
 368 {
 369         chunk_header_t chunk_header;
 370         int rnd_up_len, count;
 371         int ret;
 372
 373         /* Round up the fill length to a multiple of the block size */
 374         rnd_up_len = ALIGN(len, out->block_size);
 375
 376         /* Finally we can safely emit a chunk of data */
 377         chunk_header.chunk_type = CHUNK_TYPE_FILL;
 378         chunk_header.reserved1 = 0;
 379         chunk_header.chunk_sz = rnd_up_len / out->block_size;
 380         chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
 381         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 382
 383         if (ret < 0)
 384                 return -1;
 385         ret = out->ops->write(out, &fill_val, sizeof(fill_val));
 386         if (ret < 0)
 387                 return -1;
 388
 389         if (out->use_crc) {
 390                 count = out->block_size / sizeof(uint32_t);
 391                 while (count--)
 392                         out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
 393         }
 394
 395         out->cur_out_ptr += rnd_up_len;
 396         out->chunk_cnt++;
 397
 398         return 0;
 399 }
 400
 401 static int write_sparse_data_chunk(struct output_file *out, unsigned int len,
 402                 void *data)
 403 {
 404         chunk_header_t chunk_header;
 405         int rnd_up_len, zero_len;
 406         int ret;
 407
 408         /* Round up the data length to a multiple of the block size */
 409         rnd_up_len = ALIGN(len, out->block_size);
 410         zero_len = rnd_up_len - len;
 411
 412         /* Finally we can safely emit a chunk of data */
 413         chunk_header.chunk_type = CHUNK_TYPE_RAW;
 414         chunk_header.reserved1 = 0;
 415         chunk_header.chunk_sz = rnd_up_len / out->block_size;
 416         chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
 417         ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 418
 419         if (ret < 0)
 420                 return -1;
 421         ret = out->ops->write(out, data, len);
 422         if (ret < 0)
 423                 return -1;
 424         if (zero_len) {
 425                 ret = out->ops->write(out, out->zero_buf, zero_len);
 426                 if (ret < 0)
 427                         return -1;
 428         }
 429
 430         if (out->use_crc) {
 431                 out->crc32 = sparse_crc32(out->crc32, data, len);
 432                 if (zero_len)
 433                         out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
 434         }
 435
 436         out->cur_out_ptr += rnd_up_len;
 437         out->chunk_cnt++;
 438
 439         return 0;
 440 }
 441
 442 int write_sparse_end_chunk(struct output_file *out)
 443 {
 444         chunk_header_t chunk_header;
 445         int ret;
 446
 447         if (out->use_crc) {
 448                 chunk_header.chunk_type = CHUNK_TYPE_CRC32;
 449                 chunk_header.reserved1 = 0;
 450                 chunk_header.chunk_sz = 0;
 451                 chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
 452
 453                 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
 454                 if (ret < 0) {
 455                         return ret;
 456                 }
 457                 out->ops->write(out, &out->crc32, 4);
 458                 if (ret < 0) {
 459                         return ret;
 460                 }
 461
 462                 out->chunk_cnt++;
 463         }
 464
 465         return 0;
 466 }
 467
 468 static struct sparse_file_ops sparse_file_ops = {
 469                 .write_data_chunk = write_sparse_data_chunk,
 470                 .write_fill_chunk = write_sparse_fill_chunk,
 471                 .write_skip_chunk = write_sparse_skip_chunk,
 472                 .write_end_chunk = write_sparse_end_chunk,
 473 };
 474
 475 static int write_normal_data_chunk(struct output_file *out, unsigned int len,
 476                 void *data)
 477 {
 478         int ret;
 479         unsigned int rnd_up_len = ALIGN(len, out->block_size);
 480
 481         ret = out->ops->write(out, data, len);
 482         if (ret < 0) {
 483                 return ret;
 484         }
 485
 486         if (rnd_up_len > len) {
 487                 ret = out->ops->skip(out, rnd_up_len - len);
 488         }
 489
 490         return ret;
 491 }
 492
 493 static int write_normal_fill_chunk(struct output_file *out, unsigned int len,
 494                 uint32_t fill_val)
 495 {
 496         int ret;
 497         unsigned int i;
 498         unsigned int write_len;
 499
 500         /* Initialize fill_buf with the fill_val */
 501         for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
 502                 out->fill_buf[i] = fill_val;
 503         }
 504
 505         while (len) {
 506                 write_len = min(len, out->block_size);
 507                 ret = out->ops->write(out, out->fill_buf, write_len);
 508                 if (ret < 0) {
 509                         return ret;
 510                 }
 511
 512                 len -= write_len;
 513         }
 514
 515         return 0;
 516 }
 517
 518 static int write_normal_skip_chunk(struct output_file *out, int64_t len)
 519 {
 520         return out->ops->skip(out, len);
 521 }
 522
 523 int write_normal_end_chunk(struct output_file *out)
 524 {
 525         return out->ops->pad(out, out->len);
 526 }
 527
 528 static struct sparse_file_ops normal_file_ops = {
 529                 .write_data_chunk = write_normal_data_chunk,
 530                 .write_fill_chunk = write_normal_fill_chunk,
 531                 .write_skip_chunk = write_normal_skip_chunk,
 532                 .write_end_chunk = write_normal_end_chunk,
 533 };
 534
 535 void output_file_close(struct output_file *out)
 536 {
 537         out->sparse_ops->write_end_chunk(out);
 538         out->ops->close(out);
 539 }
 540
 541 static int output_file_init(struct output_file *out, int block_size,
 542                 int64_t len, bool sparse, int chunks, bool crc)
 543 {
 544         int ret;
 545
 546         out->len = len;
 547         out->block_size = block_size;
 548         out->cur_out_ptr = 0ll;
 549         out->chunk_cnt = 0;
 550         out->crc32 = 0;
 551         out->use_crc = crc;
 552
 553         out->zero_buf = calloc(block_size, 1);
 554         if (!out->zero_buf) {
 555                 error_errno("malloc zero_buf");
 556                 return -ENOMEM;
 557         }
 558
 559         out->fill_buf = calloc(block_size, 1);
 560         if (!out->fill_buf) {
 561                 error_errno("malloc fill_buf");
 562                 ret = -ENOMEM;
 563                 goto err_fill_buf;
 564         }
 565
 566         if (sparse) {
 567                 out->sparse_ops = &sparse_file_ops;
 568         } else {
 569                 out->sparse_ops = &normal_file_ops;
 570         }
 571
 572         if (sparse) {
 573                 sparse_header_t sparse_header = {
 574                                 .magic = SPARSE_HEADER_MAGIC,
 575                                 .major_version = SPARSE_HEADER_MAJOR_VER,
 576                                 .minor_version = SPARSE_HEADER_MINOR_VER,
 577                                 .file_hdr_sz = SPARSE_HEADER_LEN,
 578                                 .chunk_hdr_sz = CHUNK_HEADER_LEN,
 579                                 .blk_sz = out->block_size,
 580                                 .total_blks = out->len / out->block_size,
 581                                 .total_chunks = chunks,
 582                                 .image_checksum = 0
 583                 };
 584
 585                 if (out->use_crc) {
 586                         sparse_header.total_chunks++;
 587                 }
 588
 589                 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
 590                 if (ret < 0) {
 591                         goto err_write;
 592                 }
 593         }
 594
 595         return 0;
 596
 597 err_write:
 598         free(out->fill_buf);
 599 err_fill_buf:
 600         free(out->zero_buf);
 601         return ret;
 602 }
 603
 604 static struct output_file *output_file_new_gz(void)
 605 {
 606         struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz));
 607         if (!outgz) {
 608                 error_errno("malloc struct outgz");
 609                 return NULL;
 610         }
 611
 612         outgz->out.ops = &gz_file_ops;
 613
 614         return &outgz->out;
 615 }
 616
 617 static struct output_file *output_file_new_normal(void)
 618 {
 619         struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal));
 620         if (!outn) {
 621                 error_errno("malloc struct outn");
 622                 return NULL;
 623         }
 624
 625         outn->out.ops = &file_ops;
 626
 627         return &outn->out;
 628 }
 629
 630 struct output_file *output_file_open_callback(int (*write)(void *, const void *, int),
 631                 void *priv, unsigned int block_size, int64_t len,
 632                 int gz __unused, int sparse, int chunks, int crc)
 633 {
 634         int ret;
 635         struct output_file_callback *outc;
 636
 637         outc = calloc(1, sizeof(struct output_file_callback));
 638         if (!outc) {
 639                 error_errno("malloc struct outc");
 640                 return NULL;
 641         }
 642
 643         outc->out.ops = &callback_file_ops;
 644         outc->priv = priv;
 645         outc->write = write;
 646
 647         ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
 648         if (ret < 0) {
 649                 free(outc);
 650                 return NULL;
 651         }
 652
 653         return &outc->out;
 654 }
 655
 656 struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
 657                 int gz, int sparse, int chunks, int crc)
 658 {
 659         int ret;
 660         struct output_file *out;
 661
 662         if (gz) {
 663                 out = output_file_new_gz();
 664         } else {
 665                 out = output_file_new_normal();
 666         }
 667         if (!out) {
 668                 return NULL;
 669         }
 670
 671         out->ops->open(out, fd);
 672
 673         ret = output_file_init(out, block_size, len, sparse, chunks, crc);
 674         if (ret < 0) {
 675                 free(out);
 676                 return NULL;
 677         }
 678
 679         return out;
 680 }
 681
 682 /* Write a contiguous region of data blocks from a memory buffer */
 683 int write_data_chunk(struct output_file *out, unsigned int len, void *data)
 684 {
 685         return out->sparse_ops->write_data_chunk(out, len, data);
 686 }
 687
 688 /* Write a contiguous region of data blocks with a fill value */
 689 int write_fill_chunk(struct output_file *out, unsigned int len,
 690                 uint32_t fill_val)
 691 {
 692         return out->sparse_ops->write_fill_chunk(out, len, fill_val);
 693 }
 694
 695 int write_fd_chunk(struct output_file *out, unsigned int len,
 696                 int fd, int64_t offset)
 697 {
 698         int ret;
 699         int64_t aligned_offset;
 700         int aligned_diff;
 701         int buffer_size;
 702         char *ptr;
 703
 704         aligned_offset = offset & ~(4096 - 1);
 705         aligned_diff = offset - aligned_offset;
 706         buffer_size = len + aligned_diff;
 707
 708 #ifndef USE_MINGW
 709         char *data = mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
 710                         aligned_offset);
 711         if (data == MAP_FAILED) {
 712                 return -errno;
 713         }
 714         ptr = data + aligned_diff;
 715 #else
 716         off64_t pos;
 717         char *data = malloc(len);
 718         if (!data) {
 719                 return -errno;
 720         }
 721         pos = lseek64(fd, offset, SEEK_SET);
 722         if (pos < 0) {
 723                 free(data);
 724                 return -errno;
 725         }
 726         ret = read_all(fd, data, len);
 727         if (ret < 0) {
 728                 free(data);
 729                 return ret;
 730         }
 731         ptr = data;
 732 #endif
 733
 734         ret = out->sparse_ops->write_data_chunk(out, len, ptr);
 735
 736 #ifndef USE_MINGW
 737         munmap(data, buffer_size);
 738 #else
 739         free(data);
 740 #endif
 741
 742         return ret;
 743 }
 744
 745 /* Write a contiguous region of data blocks from a file */
 746 int write_file_chunk(struct output_file *out, unsigned int len,
 747                 const char *file, int64_t offset)
 748 {
 749         int ret;
 750
 751         int file_fd = open(file, O_RDONLY | O_BINARY);
 752         if (file_fd < 0) {
 753                 return -errno;
 754         }
 755
 756         ret = write_fd_chunk(out, len, file_fd, offset);
 757
 758         close(file_fd);
 759
 760         return ret;
 761 }
 762
 763 int write_skip_chunk(struct output_file *out, int64_t len)
 764 {
 765         return out->sparse_ops->write_skip_chunk(out, len);
 766 }