2 * Copyright (C) 2010 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #define _FILE_OFFSET_BITS 64
18 #define _LARGEFILE64_SOURCE 1
28 #include <sys/types.h>
33 #include "output_file.h"
34 #include "sparse_crc32.h"
35 #include "sparse_format.h"
41 #define ftruncate64 ftruncate
44 #if defined(__APPLE__) && defined(__MACH__)
46 #define ftruncate64 ftruncate
52 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
54 #define SPARSE_HEADER_MAJOR_VER 1
55 #define SPARSE_HEADER_MINOR_VER 0
56 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
57 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
59 #define container_of(inner, outer_t, elem) \
60 ((outer_t *)((char *)inner - offsetof(outer_t, elem)))
62 struct output_file_ops
{
63 int (*open
)(struct output_file
*, int fd
);
64 int (*skip
)(struct output_file
*, int64_t);
65 int (*pad
)(struct output_file
*, int64_t);
66 int (*write
)(struct output_file
*, void *, int);
67 void (*close
)(struct output_file
*);
70 struct sparse_file_ops
{
71 int (*write_data_chunk
)(struct output_file
*out
, unsigned int len
,
73 int (*write_fill_chunk
)(struct output_file
*out
, unsigned int len
,
75 int (*write_skip_chunk
)(struct output_file
*out
, int64_t len
);
76 int (*write_end_chunk
)(struct output_file
*out
);
81 unsigned int chunk_cnt
;
83 struct output_file_ops
*ops
;
84 struct sparse_file_ops
*sparse_ops
;
86 unsigned int block_size
;
93 struct output_file_gz
{
94 struct output_file out
;
98 #define to_output_file_gz(_o) \
99 container_of((_o), struct output_file_gz, out)
101 struct output_file_normal
{
102 struct output_file out
;
106 #define to_output_file_normal(_o) \
107 container_of((_o), struct output_file_normal, out)
109 struct output_file_callback
{
110 struct output_file out
;
112 int (*write
)(void *priv
, const void *buf
, int len
);
115 #define to_output_file_callback(_o) \
116 container_of((_o), struct output_file_callback, out)
118 static int file_open(struct output_file
*out
, int fd
)
120 struct output_file_normal
*outn
= to_output_file_normal(out
);
126 static int file_skip(struct output_file
*out
, int64_t cnt
)
129 struct output_file_normal
*outn
= to_output_file_normal(out
);
131 ret
= lseek64(outn
->fd
, cnt
, SEEK_CUR
);
133 error_errno("lseek64");
139 static int file_pad(struct output_file
*out
, int64_t len
)
142 struct output_file_normal
*outn
= to_output_file_normal(out
);
144 ret
= ftruncate64(outn
->fd
, len
);
152 static int file_write(struct output_file
*out
, void *data
, int len
)
155 struct output_file_normal
*outn
= to_output_file_normal(out
);
157 ret
= write(outn
->fd
, data
, len
);
159 error_errno("write");
161 } else if (ret
< len
) {
162 error("incomplete write");
169 static void file_close(struct output_file
*out
)
171 struct output_file_normal
*outn
= to_output_file_normal(out
);
176 static struct output_file_ops file_ops
= {
184 static int gz_file_open(struct output_file
*out
, int fd
)
186 struct output_file_gz
*outgz
= to_output_file_gz(out
);
188 outgz
->gz_fd
= gzdopen(fd
, "wb9");
190 error_errno("gzopen");
198 static int gz_file_skip(struct output_file
*out
, int64_t cnt
)
201 struct output_file_gz
*outgz
= to_output_file_gz(out
);
203 ret
= gzseek(outgz
->gz_fd
, cnt
, SEEK_CUR
);
205 error_errno("gzseek");
211 static int gz_file_pad(struct output_file
*out
, int64_t len
)
214 struct output_file_gz
*outgz
= to_output_file_gz(out
);
216 ret
= gztell(outgz
->gz_fd
);
225 ret
= gzseek(outgz
->gz_fd
, len
- 1, SEEK_SET
);
230 gzwrite(outgz
->gz_fd
, "", 1);
235 static int gz_file_write(struct output_file
*out
, void *data
, int len
)
238 struct output_file_gz
*outgz
= to_output_file_gz(out
);
240 ret
= gzwrite(outgz
->gz_fd
, data
, len
);
242 error_errno("gzwrite");
244 } else if (ret
< len
) {
245 error("incomplete gzwrite");
252 static void gz_file_close(struct output_file
*out
)
254 struct output_file_gz
*outgz
= to_output_file_gz(out
);
256 gzclose(outgz
->gz_fd
);
260 static struct output_file_ops gz_file_ops
= {
261 .open
= gz_file_open
,
262 .skip
= gz_file_skip
,
264 .write
= gz_file_write
,
265 .close
= gz_file_close
,
268 static int callback_file_open(struct output_file
*out __unused
, int fd __unused
)
273 static int callback_file_skip(struct output_file
*out
, int64_t off
)
275 struct output_file_callback
*outc
= to_output_file_callback(out
);
280 to_write
= min(off
, (int64_t)INT_MAX
);
281 ret
= outc
->write(outc
->priv
, NULL
, to_write
);
291 static int callback_file_pad(struct output_file
*out __unused
, int64_t len __unused
)
296 static int callback_file_write(struct output_file
*out
, void *data
, int len
)
298 struct output_file_callback
*outc
= to_output_file_callback(out
);
300 return outc
->write(outc
->priv
, data
, len
);
303 static void callback_file_close(struct output_file
*out
)
305 struct output_file_callback
*outc
= to_output_file_callback(out
);
310 static struct output_file_ops callback_file_ops
= {
311 .open
= callback_file_open
,
312 .skip
= callback_file_skip
,
313 .pad
= callback_file_pad
,
314 .write
= callback_file_write
,
315 .close
= callback_file_close
,
318 int read_all(int fd
, void *buf
, size_t len
)
324 while (total
< len
) {
325 ret
= read(fd
, ptr
, len
- total
);
340 static int write_sparse_skip_chunk(struct output_file
*out
, int64_t skip_len
)
342 chunk_header_t chunk_header
;
345 if (skip_len
% out
->block_size
) {
346 error("don't care size %"PRIi64
" is not a multiple of the block size %u",
347 skip_len
, out
->block_size
);
351 /* We are skipping data, so emit a don't care chunk. */
352 chunk_header
.chunk_type
= CHUNK_TYPE_DONT_CARE
;
353 chunk_header
.reserved1
= 0;
354 chunk_header
.chunk_sz
= skip_len
/ out
->block_size
;
355 chunk_header
.total_sz
= CHUNK_HEADER_LEN
;
356 ret
= out
->ops
->write(out
, &chunk_header
, sizeof(chunk_header
));
360 out
->cur_out_ptr
+= skip_len
;
366 static int write_sparse_fill_chunk(struct output_file
*out
, unsigned int len
,
369 chunk_header_t chunk_header
;
370 int rnd_up_len
, count
;
373 /* Round up the fill length to a multiple of the block size */
374 rnd_up_len
= ALIGN(len
, out
->block_size
);
376 /* Finally we can safely emit a chunk of data */
377 chunk_header
.chunk_type
= CHUNK_TYPE_FILL
;
378 chunk_header
.reserved1
= 0;
379 chunk_header
.chunk_sz
= rnd_up_len
/ out
->block_size
;
380 chunk_header
.total_sz
= CHUNK_HEADER_LEN
+ sizeof(fill_val
);
381 ret
= out
->ops
->write(out
, &chunk_header
, sizeof(chunk_header
));
385 ret
= out
->ops
->write(out
, &fill_val
, sizeof(fill_val
));
390 count
= out
->block_size
/ sizeof(uint32_t);
392 out
->crc32
= sparse_crc32(out
->crc32
, &fill_val
, sizeof(uint32_t));
395 out
->cur_out_ptr
+= rnd_up_len
;
401 static int write_sparse_data_chunk(struct output_file
*out
, unsigned int len
,
404 chunk_header_t chunk_header
;
405 int rnd_up_len
, zero_len
;
408 /* Round up the data length to a multiple of the block size */
409 rnd_up_len
= ALIGN(len
, out
->block_size
);
410 zero_len
= rnd_up_len
- len
;
412 /* Finally we can safely emit a chunk of data */
413 chunk_header
.chunk_type
= CHUNK_TYPE_RAW
;
414 chunk_header
.reserved1
= 0;
415 chunk_header
.chunk_sz
= rnd_up_len
/ out
->block_size
;
416 chunk_header
.total_sz
= CHUNK_HEADER_LEN
+ rnd_up_len
;
417 ret
= out
->ops
->write(out
, &chunk_header
, sizeof(chunk_header
));
421 ret
= out
->ops
->write(out
, data
, len
);
425 ret
= out
->ops
->write(out
, out
->zero_buf
, zero_len
);
431 out
->crc32
= sparse_crc32(out
->crc32
, data
, len
);
433 out
->crc32
= sparse_crc32(out
->crc32
, out
->zero_buf
, zero_len
);
436 out
->cur_out_ptr
+= rnd_up_len
;
442 int write_sparse_end_chunk(struct output_file
*out
)
444 chunk_header_t chunk_header
;
448 chunk_header
.chunk_type
= CHUNK_TYPE_CRC32
;
449 chunk_header
.reserved1
= 0;
450 chunk_header
.chunk_sz
= 0;
451 chunk_header
.total_sz
= CHUNK_HEADER_LEN
+ 4;
453 ret
= out
->ops
->write(out
, &chunk_header
, sizeof(chunk_header
));
457 out
->ops
->write(out
, &out
->crc32
, 4);
468 static struct sparse_file_ops sparse_file_ops
= {
469 .write_data_chunk
= write_sparse_data_chunk
,
470 .write_fill_chunk
= write_sparse_fill_chunk
,
471 .write_skip_chunk
= write_sparse_skip_chunk
,
472 .write_end_chunk
= write_sparse_end_chunk
,
475 static int write_normal_data_chunk(struct output_file
*out
, unsigned int len
,
479 unsigned int rnd_up_len
= ALIGN(len
, out
->block_size
);
481 ret
= out
->ops
->write(out
, data
, len
);
486 if (rnd_up_len
> len
) {
487 ret
= out
->ops
->skip(out
, rnd_up_len
- len
);
493 static int write_normal_fill_chunk(struct output_file
*out
, unsigned int len
,
498 unsigned int write_len
;
500 /* Initialize fill_buf with the fill_val */
501 for (i
= 0; i
< out
->block_size
/ sizeof(uint32_t); i
++) {
502 out
->fill_buf
[i
] = fill_val
;
506 write_len
= min(len
, out
->block_size
);
507 ret
= out
->ops
->write(out
, out
->fill_buf
, write_len
);
518 static int write_normal_skip_chunk(struct output_file
*out
, int64_t len
)
520 return out
->ops
->skip(out
, len
);
523 int write_normal_end_chunk(struct output_file
*out
)
525 return out
->ops
->pad(out
, out
->len
);
528 static struct sparse_file_ops normal_file_ops
= {
529 .write_data_chunk
= write_normal_data_chunk
,
530 .write_fill_chunk
= write_normal_fill_chunk
,
531 .write_skip_chunk
= write_normal_skip_chunk
,
532 .write_end_chunk
= write_normal_end_chunk
,
535 void output_file_close(struct output_file
*out
)
537 out
->sparse_ops
->write_end_chunk(out
);
538 out
->ops
->close(out
);
541 static int output_file_init(struct output_file
*out
, int block_size
,
542 int64_t len
, bool sparse
, int chunks
, bool crc
)
547 out
->block_size
= block_size
;
548 out
->cur_out_ptr
= 0ll;
553 out
->zero_buf
= calloc(block_size
, 1);
554 if (!out
->zero_buf
) {
555 error_errno("malloc zero_buf");
559 out
->fill_buf
= calloc(block_size
, 1);
560 if (!out
->fill_buf
) {
561 error_errno("malloc fill_buf");
567 out
->sparse_ops
= &sparse_file_ops
;
569 out
->sparse_ops
= &normal_file_ops
;
573 sparse_header_t sparse_header
= {
574 .magic
= SPARSE_HEADER_MAGIC
,
575 .major_version
= SPARSE_HEADER_MAJOR_VER
,
576 .minor_version
= SPARSE_HEADER_MINOR_VER
,
577 .file_hdr_sz
= SPARSE_HEADER_LEN
,
578 .chunk_hdr_sz
= CHUNK_HEADER_LEN
,
579 .blk_sz
= out
->block_size
,
580 .total_blks
= out
->len
/ out
->block_size
,
581 .total_chunks
= chunks
,
586 sparse_header
.total_chunks
++;
589 ret
= out
->ops
->write(out
, &sparse_header
, sizeof(sparse_header
));
604 static struct output_file
*output_file_new_gz(void)
606 struct output_file_gz
*outgz
= calloc(1, sizeof(struct output_file_gz
));
608 error_errno("malloc struct outgz");
612 outgz
->out
.ops
= &gz_file_ops
;
617 static struct output_file
*output_file_new_normal(void)
619 struct output_file_normal
*outn
= calloc(1, sizeof(struct output_file_normal
));
621 error_errno("malloc struct outn");
625 outn
->out
.ops
= &file_ops
;
630 struct output_file
*output_file_open_callback(int (*write
)(void *, const void *, int),
631 void *priv
, unsigned int block_size
, int64_t len
,
632 int gz __unused
, int sparse
, int chunks
, int crc
)
635 struct output_file_callback
*outc
;
637 outc
= calloc(1, sizeof(struct output_file_callback
));
639 error_errno("malloc struct outc");
643 outc
->out
.ops
= &callback_file_ops
;
647 ret
= output_file_init(&outc
->out
, block_size
, len
, sparse
, chunks
, crc
);
656 struct output_file
*output_file_open_fd(int fd
, unsigned int block_size
, int64_t len
,
657 int gz
, int sparse
, int chunks
, int crc
)
660 struct output_file
*out
;
663 out
= output_file_new_gz();
665 out
= output_file_new_normal();
671 out
->ops
->open(out
, fd
);
673 ret
= output_file_init(out
, block_size
, len
, sparse
, chunks
, crc
);
682 /* Write a contiguous region of data blocks from a memory buffer */
683 int write_data_chunk(struct output_file
*out
, unsigned int len
, void *data
)
685 return out
->sparse_ops
->write_data_chunk(out
, len
, data
);
688 /* Write a contiguous region of data blocks with a fill value */
689 int write_fill_chunk(struct output_file
*out
, unsigned int len
,
692 return out
->sparse_ops
->write_fill_chunk(out
, len
, fill_val
);
695 int write_fd_chunk(struct output_file
*out
, unsigned int len
,
696 int fd
, int64_t offset
)
699 int64_t aligned_offset
;
704 aligned_offset
= offset
& ~(4096 - 1);
705 aligned_diff
= offset
- aligned_offset
;
706 buffer_size
= len
+ aligned_diff
;
709 char *data
= mmap64(NULL
, buffer_size
, PROT_READ
, MAP_SHARED
, fd
,
711 if (data
== MAP_FAILED
) {
714 ptr
= data
+ aligned_diff
;
717 char *data
= malloc(len
);
721 pos
= lseek64(fd
, offset
, SEEK_SET
);
726 ret
= read_all(fd
, data
, len
);
734 ret
= out
->sparse_ops
->write_data_chunk(out
, len
, ptr
);
737 munmap(data
, buffer_size
);
745 /* Write a contiguous region of data blocks from a file */
746 int write_file_chunk(struct output_file
*out
, unsigned int len
,
747 const char *file
, int64_t offset
)
751 int file_fd
= open(file
, O_RDONLY
| O_BINARY
);
756 ret
= write_fd_chunk(out
, len
, file_fd
, offset
);
763 int write_skip_chunk(struct output_file
*out
, int64_t len
)
765 return out
->sparse_ops
->write_skip_chunk(out
, len
);