2 * Copyright (C) 2010 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #define _FILE_OFFSET_BITS 64
18 #define _LARGEFILE64_SOURCE 1
29 #include <sys/types.h>
34 #include "output_file.h"
35 #include "sparse_crc32.h"
36 #include "sparse_format.h"
39 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
41 #define SPARSE_HEADER_MAJOR_VER 1
42 #define SPARSE_HEADER_MINOR_VER 0
43 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
44 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
46 #define container_of(inner, outer_t, elem) \
47 ((outer_t *)((char *)inner - offsetof(outer_t, elem)))
49 struct output_file_ops
{
50 int (*open
)(struct output_file
*, int fd
);
51 int (*skip
)(struct output_file
*, int64_t);
52 int (*pad
)(struct output_file
*, int64_t);
53 int (*write
)(struct output_file
*, void *, int);
54 void (*close
)(struct output_file
*);
57 struct sparse_file_ops
{
58 int (*write_data_chunk
)(struct output_file
*out
, unsigned int len
,
60 int (*write_fill_chunk
)(struct output_file
*out
, unsigned int len
,
62 int (*write_skip_chunk
)(struct output_file
*out
, int64_t len
);
63 int (*write_end_chunk
)(struct output_file
*out
);
68 unsigned int chunk_cnt
;
70 struct output_file_ops
*ops
;
71 struct sparse_file_ops
*sparse_ops
;
73 unsigned int block_size
;
80 struct output_file_gz
{
81 struct output_file out
;
85 #define to_output_file_gz(_o) \
86 container_of((_o), struct output_file_gz, out)
88 struct output_file_normal
{
89 struct output_file out
;
93 #define to_output_file_normal(_o) \
94 container_of((_o), struct output_file_normal, out)
96 struct output_file_callback
{
97 struct output_file out
;
99 int (*write
)(void *priv
, const void *buf
, int len
);
102 #define to_output_file_callback(_o) \
103 container_of((_o), struct output_file_callback, out)
105 static int file_open(struct output_file
*out
, int fd
)
107 struct output_file_normal
*outn
= to_output_file_normal(out
);
113 static int file_skip(struct output_file
*out
, int64_t cnt
)
116 struct output_file_normal
*outn
= to_output_file_normal(out
);
118 ret
= lseek(outn
->fd
, cnt
, SEEK_CUR
);
120 error_errno("lseek");
126 static int file_pad(struct output_file
*out
, int64_t len
)
129 struct output_file_normal
*outn
= to_output_file_normal(out
);
131 ret
= ftruncate(outn
->fd
, len
);
139 static int file_write(struct output_file
*out
, void *data
, int len
)
142 struct output_file_normal
*outn
= to_output_file_normal(out
);
144 ret
= write(outn
->fd
, data
, len
);
146 error_errno("write");
148 } else if (ret
< len
) {
149 error("incomplete write");
156 static void file_close(struct output_file
*out
)
158 struct output_file_normal
*outn
= to_output_file_normal(out
);
163 static struct output_file_ops file_ops
= {
171 static int gz_file_open(struct output_file
*out
, int fd
)
173 struct output_file_gz
*outgz
= to_output_file_gz(out
);
175 outgz
->gz_fd
= gzdopen(fd
, "wb9");
177 error_errno("gzopen");
185 static int gz_file_skip(struct output_file
*out
, int64_t cnt
)
188 struct output_file_gz
*outgz
= to_output_file_gz(out
);
190 ret
= gzseek(outgz
->gz_fd
, cnt
, SEEK_CUR
);
192 error_errno("gzseek");
198 static int gz_file_pad(struct output_file
*out
, int64_t len
)
201 struct output_file_gz
*outgz
= to_output_file_gz(out
);
203 ret
= gztell(outgz
->gz_fd
);
212 ret
= gzseek(outgz
->gz_fd
, len
- 1, SEEK_SET
);
217 gzwrite(outgz
->gz_fd
, "", 1);
222 static int gz_file_write(struct output_file
*out
, void *data
, int len
)
225 struct output_file_gz
*outgz
= to_output_file_gz(out
);
227 ret
= gzwrite(outgz
->gz_fd
, data
, len
);
229 error_errno("gzwrite");
231 } else if (ret
< len
) {
232 error("incomplete gzwrite");
239 static void gz_file_close(struct output_file
*out
)
241 struct output_file_gz
*outgz
= to_output_file_gz(out
);
243 gzclose(outgz
->gz_fd
);
247 static struct output_file_ops gz_file_ops
= {
248 .open
= gz_file_open
,
249 .skip
= gz_file_skip
,
251 .write
= gz_file_write
,
252 .close
= gz_file_close
,
255 static int callback_file_open(struct output_file
*out __unused
, int fd __unused
)
260 static int callback_file_skip(struct output_file
*out
, int64_t off
)
262 struct output_file_callback
*outc
= to_output_file_callback(out
);
267 to_write
= min(off
, (int64_t)INT_MAX
);
268 ret
= outc
->write(outc
->priv
, NULL
, to_write
);
278 static int callback_file_pad(struct output_file
*out __unused
, int64_t len __unused
)
283 static int callback_file_write(struct output_file
*out
, void *data
, int len
)
285 struct output_file_callback
*outc
= to_output_file_callback(out
);
287 return outc
->write(outc
->priv
, data
, len
);
290 static void callback_file_close(struct output_file
*out
)
292 struct output_file_callback
*outc
= to_output_file_callback(out
);
297 static struct output_file_ops callback_file_ops
= {
298 .open
= callback_file_open
,
299 .skip
= callback_file_skip
,
300 .pad
= callback_file_pad
,
301 .write
= callback_file_write
,
302 .close
= callback_file_close
,
305 int read_all(int fd
, void *buf
, size_t len
)
311 while (total
< len
) {
312 ret
= read(fd
, ptr
, len
- total
);
327 static int write_sparse_skip_chunk(struct output_file
*out
, int64_t skip_len
)
329 chunk_header_t chunk_header
;
332 if (skip_len
% out
->block_size
) {
333 error("don't care size %"PRIi64
" is not a multiple of the block size %u",
334 skip_len
, out
->block_size
);
338 /* We are skipping data, so emit a don't care chunk. */
339 chunk_header
.chunk_type
= CHUNK_TYPE_DONT_CARE
;
340 chunk_header
.reserved1
= 0;
341 chunk_header
.chunk_sz
= skip_len
/ out
->block_size
;
342 chunk_header
.total_sz
= CHUNK_HEADER_LEN
;
343 ret
= out
->ops
->write(out
, &chunk_header
, sizeof(chunk_header
));
347 out
->cur_out_ptr
+= skip_len
;
353 static int write_sparse_fill_chunk(struct output_file
*out
, unsigned int len
,
356 chunk_header_t chunk_header
;
357 int rnd_up_len
, count
;
360 /* Round up the fill length to a multiple of the block size */
361 rnd_up_len
= ALIGN(len
, out
->block_size
);
363 /* Finally we can safely emit a chunk of data */
364 chunk_header
.chunk_type
= CHUNK_TYPE_FILL
;
365 chunk_header
.reserved1
= 0;
366 chunk_header
.chunk_sz
= rnd_up_len
/ out
->block_size
;
367 chunk_header
.total_sz
= CHUNK_HEADER_LEN
+ sizeof(fill_val
);
368 ret
= out
->ops
->write(out
, &chunk_header
, sizeof(chunk_header
));
372 ret
= out
->ops
->write(out
, &fill_val
, sizeof(fill_val
));
377 count
= out
->block_size
/ sizeof(uint32_t);
379 out
->crc32
= sparse_crc32(out
->crc32
, &fill_val
, sizeof(uint32_t));
382 out
->cur_out_ptr
+= rnd_up_len
;
388 static int write_sparse_data_chunk(struct output_file
*out
, unsigned int len
,
391 chunk_header_t chunk_header
;
392 int rnd_up_len
, zero_len
;
395 /* Round up the data length to a multiple of the block size */
396 rnd_up_len
= ALIGN(len
, out
->block_size
);
397 zero_len
= rnd_up_len
- len
;
399 /* Finally we can safely emit a chunk of data */
400 chunk_header
.chunk_type
= CHUNK_TYPE_RAW
;
401 chunk_header
.reserved1
= 0;
402 chunk_header
.chunk_sz
= rnd_up_len
/ out
->block_size
;
403 chunk_header
.total_sz
= CHUNK_HEADER_LEN
+ rnd_up_len
;
404 ret
= out
->ops
->write(out
, &chunk_header
, sizeof(chunk_header
));
408 ret
= out
->ops
->write(out
, data
, len
);
412 ret
= out
->ops
->write(out
, out
->zero_buf
, zero_len
);
418 out
->crc32
= sparse_crc32(out
->crc32
, data
, len
);
420 out
->crc32
= sparse_crc32(out
->crc32
, out
->zero_buf
, zero_len
);
423 out
->cur_out_ptr
+= rnd_up_len
;
429 int write_sparse_end_chunk(struct output_file
*out
)
431 chunk_header_t chunk_header
;
435 chunk_header
.chunk_type
= CHUNK_TYPE_CRC32
;
436 chunk_header
.reserved1
= 0;
437 chunk_header
.chunk_sz
= 0;
438 chunk_header
.total_sz
= CHUNK_HEADER_LEN
+ 4;
440 ret
= out
->ops
->write(out
, &chunk_header
, sizeof(chunk_header
));
444 out
->ops
->write(out
, &out
->crc32
, 4);
455 static struct sparse_file_ops sparse_file_ops
= {
456 .write_data_chunk
= write_sparse_data_chunk
,
457 .write_fill_chunk
= write_sparse_fill_chunk
,
458 .write_skip_chunk
= write_sparse_skip_chunk
,
459 .write_end_chunk
= write_sparse_end_chunk
,
462 static int write_normal_data_chunk(struct output_file
*out
, unsigned int len
,
466 unsigned int rnd_up_len
= ALIGN(len
, out
->block_size
);
468 ret
= out
->ops
->write(out
, data
, len
);
473 if (rnd_up_len
> len
) {
474 ret
= out
->ops
->skip(out
, rnd_up_len
- len
);
480 static int write_normal_fill_chunk(struct output_file
*out
, unsigned int len
,
485 unsigned int write_len
;
487 /* Initialize fill_buf with the fill_val */
488 for (i
= 0; i
< out
->block_size
/ sizeof(uint32_t); i
++) {
489 out
->fill_buf
[i
] = fill_val
;
493 write_len
= min(len
, out
->block_size
);
494 ret
= out
->ops
->write(out
, out
->fill_buf
, write_len
);
505 static int write_normal_skip_chunk(struct output_file
*out
, int64_t len
)
507 return out
->ops
->skip(out
, len
);
510 int write_normal_end_chunk(struct output_file
*out
)
512 return out
->ops
->pad(out
, out
->len
);
515 static struct sparse_file_ops normal_file_ops
= {
516 .write_data_chunk
= write_normal_data_chunk
,
517 .write_fill_chunk
= write_normal_fill_chunk
,
518 .write_skip_chunk
= write_normal_skip_chunk
,
519 .write_end_chunk
= write_normal_end_chunk
,
522 void output_file_close(struct output_file
*out
)
524 out
->sparse_ops
->write_end_chunk(out
);
525 out
->ops
->close(out
);
528 static int output_file_init(struct output_file
*out
, int block_size
,
529 int64_t len
, bool sparse
, int chunks
, bool crc
)
534 out
->block_size
= block_size
;
535 out
->cur_out_ptr
= 0ll;
540 out
->zero_buf
= calloc(block_size
, 1);
541 if (!out
->zero_buf
) {
542 error_errno("malloc zero_buf");
546 out
->fill_buf
= calloc(block_size
, 1);
547 if (!out
->fill_buf
) {
548 error_errno("malloc fill_buf");
554 out
->sparse_ops
= &sparse_file_ops
;
556 out
->sparse_ops
= &normal_file_ops
;
560 sparse_header_t sparse_header
= {
561 .magic
= SPARSE_HEADER_MAGIC
,
562 .major_version
= SPARSE_HEADER_MAJOR_VER
,
563 .minor_version
= SPARSE_HEADER_MINOR_VER
,
564 .file_hdr_sz
= SPARSE_HEADER_LEN
,
565 .chunk_hdr_sz
= CHUNK_HEADER_LEN
,
566 .blk_sz
= out
->block_size
,
567 .total_blks
= out
->len
/ out
->block_size
,
568 .total_chunks
= chunks
,
573 sparse_header
.total_chunks
++;
576 ret
= out
->ops
->write(out
, &sparse_header
, sizeof(sparse_header
));
591 static struct output_file
*output_file_new_gz(void)
593 struct output_file_gz
*outgz
= calloc(1, sizeof(struct output_file_gz
));
595 error_errno("malloc struct outgz");
599 outgz
->out
.ops
= &gz_file_ops
;
604 static struct output_file
*output_file_new_normal(void)
606 struct output_file_normal
*outn
= calloc(1, sizeof(struct output_file_normal
));
608 error_errno("malloc struct outn");
612 outn
->out
.ops
= &file_ops
;
617 struct output_file
*output_file_open_callback(int (*write
)(void *, const void *, int),
618 void *priv
, unsigned int block_size
, int64_t len
,
619 int gz __unused
, int sparse
, int chunks
, int crc
)
622 struct output_file_callback
*outc
;
624 outc
= calloc(1, sizeof(struct output_file_callback
));
626 error_errno("malloc struct outc");
630 outc
->out
.ops
= &callback_file_ops
;
634 ret
= output_file_init(&outc
->out
, block_size
, len
, sparse
, chunks
, crc
);
643 struct output_file
*output_file_open_fd(int fd
, unsigned int block_size
, int64_t len
,
644 int gz
, int sparse
, int chunks
, int crc
)
647 struct output_file
*out
;
650 out
= output_file_new_gz();
652 out
= output_file_new_normal();
658 out
->ops
->open(out
, fd
);
660 ret
= output_file_init(out
, block_size
, len
, sparse
, chunks
, crc
);
669 /* Write a contiguous region of data blocks from a memory buffer */
670 int write_data_chunk(struct output_file
*out
, unsigned int len
, void *data
)
672 return out
->sparse_ops
->write_data_chunk(out
, len
, data
);
675 /* Write a contiguous region of data blocks with a fill value */
676 int write_fill_chunk(struct output_file
*out
, unsigned int len
,
679 return out
->sparse_ops
->write_fill_chunk(out
, len
, fill_val
);
682 int write_fd_chunk(struct output_file
*out
, unsigned int len
,
683 int fd
, int64_t offset
)
686 int64_t aligned_offset
;
691 aligned_offset
= offset
& ~(4096 - 1);
692 aligned_diff
= offset
- aligned_offset
;
693 buffer_size
= len
+ aligned_diff
;
695 char *data
= mmap(NULL
, buffer_size
, PROT_READ
, MAP_SHARED
, fd
,
697 if (data
== MAP_FAILED
) {
700 ptr
= data
+ aligned_diff
;
702 ret
= out
->sparse_ops
->write_data_chunk(out
, len
, ptr
);
704 munmap(data
, buffer_size
);
709 /* Write a contiguous region of data blocks from a file */
710 int write_file_chunk(struct output_file
*out
, unsigned int len
,
711 const char *file
, int64_t offset
)
715 int file_fd
= open(file
, O_RDONLY
);
720 ret
= write_fd_chunk(out
, len
, file_fd
, offset
);
727 int write_skip_chunk(struct output_file
*out
, int64_t len
)
729 return out
->sparse_ops
->write_skip_chunk(out
, len
);