projects / project / uclient.git / blob
? search:


359715c2db9bf87d0e286ced841bdf71b807acd8
[project/uclient.git] / uclient-fetch.c
1 /*
2 * uclient - ustream based protocol client library
3 *
4 * Copyright (C) 2014 Felix Fietkau <nbd@openwrt.org>
5 *
6 * Permission to use, copy, modify, and/or distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #define _GNU_SOURCE
20 #include <sys/stat.h>
21 #include <sys/socket.h>
22 #include <unistd.h>
23 #include <stdio.h>
24 #include <getopt.h>
25 #include <fcntl.h>
26 #include <glob.h>
27 #include <stdint.h>
28 #include <inttypes.h>
29 #include <signal.h>
30
31 #include <libubox/blobmsg.h>
32
33 #include "progress.h"
34 #include "uclient.h"
35 #include "uclient-utils.h"
36
37 #ifndef strdupa
38 #define strdupa(x) strcpy(alloca(strlen(x)+1),x)
39 #endif
40
41 static const char *user_agent = "uclient-fetch";
42 static const char *post_data;
43 static const char *post_file;
44 static struct ustream_ssl_ctx *ssl_ctx;
45 static const struct ustream_ssl_ops *ssl_ops;
46 static int quiet = false;
47 static bool verify = true;
48 static bool proxy = true;
49 static bool default_certs = false;
50 static bool no_output;
51 static const char *opt_output_file;
52 static int output_fd = -1;
53 static int error_ret;
54 static off_t out_offset;
55 static off_t out_bytes;
56 static off_t out_len;
57 static char *auth_str;
58 static char **urls;
59 static int n_urls;
60 static int timeout;
61 static bool resume, cur_resume;
62
63 static struct progress pmt;
64 static struct uloop_timeout pmt_timer;
65
66 static int init_request(struct uclient *cl);
67 static void request_done(struct uclient *cl);
68
69 static void pmt_update(struct uloop_timeout *t)
70 {
71 progress_update(&pmt, out_offset, out_bytes, out_len);
72 uloop_timeout_set(t, 1000);
73 }
74
75 static const char *
76 get_proxy_url(char *url)
77 {
78 char prefix[16];
79 char *sep;
80
81 if (!proxy)
82 return NULL;
83
84 sep = strchr(url, ':');
85 if (!sep)
86 return NULL;
87
88 if (sep - url > 5)
89 return NULL;
90
91 memcpy(prefix, url, sep - url);
92 strcpy(prefix + (sep - url), "_proxy");
93 return getenv(prefix);
94 }
95
96 static int open_output_file(const char *path, uint64_t resume_offset)
97 {
98 const char *output_file = opt_output_file;
99 char *filename = NULL;
100 int flags;
101 int ret;
102
103 if (cur_resume)
104 flags = O_RDWR;
105 else
106 flags = O_WRONLY | O_TRUNC;
107
108 if (!cur_resume && !output_file)
109 flags |= O_EXCL;
110
111 flags |= O_CREAT;
112
113 if (output_file) {
114 if (!strcmp(output_file, "-")) {
115 if (!quiet)
116 fprintf(stderr, "Writing to stdout\n");
117
118 ret = STDOUT_FILENO;
119 goto done;
120 }
121 } else {
122 filename = uclient_get_url_filename(path, "index.html");
123 if (!filename) {
124 ret = -ENOMEM;
125 goto out;
126 }
127
128 output_file = filename;
129 }
130
131 if (!quiet)
132 fprintf(stderr, "Writing to '%s'\n", output_file);
133 ret = open(output_file, flags, 0644);
134 if (ret < 0)
135 goto free;
136
137 if (resume_offset &&
138 lseek(ret, resume_offset, SEEK_SET) < 0) {
139 if (!quiet)
140 fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset);
141 close(ret);
142 ret = -1;
143 goto free;
144 }
145
146 out_offset = resume_offset;
147 out_bytes += resume_offset;
148 done:
149 if (!quiet) {
150 progress_init(&pmt, output_file);
151 pmt_timer.cb = pmt_update;
152 pmt_timer.cb(&pmt_timer);
153 }
154
155 free:
156 free(filename);
157 out:
158 return ret;
159 }
160
161 static void header_done_cb(struct uclient *cl)
162 {
163 enum {
164 H_RANGE,
165 H_LEN,
166 __H_MAX
167 };
168 static const struct blobmsg_policy policy[__H_MAX] = {
169 [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING },
170 [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING },
171 };
172 struct blob_attr *tb[__H_MAX];
173 uint64_t resume_offset = 0, resume_end, resume_size;
174 static int retries;
175
176 if (retries < 10) {
177 int ret = uclient_http_redirect(cl);
178 if (ret < 0) {
179 if (!quiet)
180 fprintf(stderr, "Failed to redirect to %s on %s\n", cl->url->location, cl->url->host);
181 error_ret = 8;
182 request_done(cl);
183 return;
184 }
185 if (ret > 0) {
186 if (!quiet)
187 fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host);
188
189 retries++;
190 return;
191 }
192 }
193
194 if (cl->status_code == 204 && cur_resume) {
195 /* Resume attempt failed, try normal download */
196 cur_resume = false;
197 init_request(cl);
198 return;
199 }
200
201 blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta));
202
203 switch (cl->status_code) {
204 case 416:
205 if (!quiet)
206 fprintf(stderr, "File download already fully retrieved; nothing to do.\n");
207 request_done(cl);
208 break;
209 case 206:
210 if (!cur_resume) {
211 if (!quiet)
212 fprintf(stderr, "Error: Partial content received, full content requested\n");
213 error_ret = 8;
214 request_done(cl);
215 break;
216 }
217
218 if (!tb[H_RANGE]) {
219 if (!quiet)
220 fprintf(stderr, "Content-Range header is missing\n");
221 error_ret = 8;
222 break;
223 }
224
225 if (sscanf(blobmsg_get_string(tb[H_RANGE]),
226 "bytes %"PRIu64"-%"PRIu64"/%"PRIu64,
227 &resume_offset, &resume_end, &resume_size) != 3) {
228 if (!quiet)
229 fprintf(stderr, "Content-Range header is invalid\n");
230 error_ret = 8;
231 break;
232 }
233 /* fall through */
234 case 204:
235 case 200:
236 if (no_output)
237 break;
238
239 if (tb[H_LEN])
240 out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10);
241
242 output_fd = open_output_file(cl->url->location, resume_offset);
243 if (output_fd < 0) {
244 if (!quiet)
245 perror("Cannot open output file");
246 error_ret = 3;
247 request_done(cl);
248 }
249 break;
250
251 default:
252 if (!quiet)
253 fprintf(stderr, "HTTP error %d\n", cl->status_code);
254 request_done(cl);
255 error_ret = 8;
256 break;
257 }
258 }
259
260 static void read_data_cb(struct uclient *cl)
261 {
262 char buf[256];
263 ssize_t n;
264 int len;
265
266 if (!no_output && output_fd < 0)
267 return;
268
269 while (1) {
270 len = uclient_read(cl, buf, sizeof(buf));
271 if (len <= 0)
272 return;
273
274 out_bytes += len;
275 if (!no_output) {
276 n = write(output_fd, buf, len);
277 if (n < 0)
278 return;
279 }
280 }
281 }
282
283 static void msg_connecting(struct uclient *cl)
284 {
285 char addr[INET6_ADDRSTRLEN];
286 int port;
287
288 if (quiet)
289 return;
290
291 uclient_get_addr(addr, &port, &cl->remote_addr);
292 fprintf(stderr, "Connecting to %s:%d\n", addr, port);
293 }
294
295 static void check_resume_offset(struct uclient *cl)
296 {
297 char range_str[64];
298 struct stat st;
299 char *file;
300 int ret;
301
302 file = uclient_get_url_filename(cl->url->location, "index.html");
303 if (!file)
304 return;
305
306 ret = stat(file, &st);
307 free(file);
308 if (ret)
309 return;
310
311 if (!st.st_size)
312 return;
313
314 snprintf(range_str, sizeof(range_str), "bytes=%"PRIu64"-", (uint64_t) st.st_size);
315 uclient_http_set_header(cl, "Range", range_str);
316 }
317
318 static int init_request(struct uclient *cl)
319 {
320 int rc;
321
322 out_offset = 0;
323 out_bytes = 0;
324 out_len = 0;
325 uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify);
326
327 if (timeout)
328 cl->timeout_msecs = timeout * 1000;
329
330 rc = uclient_connect(cl);
331 if (rc)
332 return rc;
333
334 msg_connecting(cl);
335
336 rc = uclient_http_set_request_type(cl, post_data || post_file ? "POST" : "GET");
337 if (rc)
338 return rc;
339
340 uclient_http_reset_headers(cl);
341 uclient_http_set_header(cl, "User-Agent", user_agent);
342 if (cur_resume)
343 check_resume_offset(cl);
344
345 if (post_data) {
346 uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded");
347 uclient_write(cl, post_data, strlen(post_data));
348 }
349 else if(post_file)
350 {
351 FILE *input_file;
352 uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded");
353
354 input_file = fopen(post_file, "r");
355 if (!input_file)
356 return errno;
357
358 char tbuf[1024];
359 size_t rlen = 0;
360 do
361 {
362 rlen = fread(tbuf, 1, sizeof(tbuf), input_file);
363 uclient_write(cl, tbuf, rlen);
364 }
365 while(rlen);
366
367 fclose(input_file);
368 }
369
370 rc = uclient_request(cl);
371 if (rc)
372 return rc;
373
374 return 0;
375 }
376
377 static void request_done(struct uclient *cl)
378 {
379 const char *proxy_url;
380
381 if (n_urls) {
382 proxy_url = get_proxy_url(*urls);
383 if (proxy_url) {
384 uclient_set_url(cl, proxy_url, NULL);
385 uclient_set_proxy_url(cl, *urls, auth_str);
386 } else {
387 uclient_set_url(cl, *urls, auth_str);
388 }
389 n_urls--;
390 cur_resume = resume;
391 error_ret = init_request(cl);
392 if (error_ret == 0)
393 return;
394 }
395
396 if (output_fd >= 0 && !opt_output_file) {
397 close(output_fd);
398 output_fd = -1;
399 }
400 uclient_disconnect(cl);
401 uloop_end();
402 }
403
404
405 static void eof_cb(struct uclient *cl)
406 {
407 if (!quiet) {
408 pmt_update(&pmt_timer);
409 uloop_timeout_cancel(&pmt_timer);
410 fprintf(stderr, "\n");
411 }
412
413 if (!cl->data_eof) {
414 if (!quiet)
415 fprintf(stderr, "Connection reset prematurely\n");
416 error_ret = 4;
417 } else if (!quiet) {
418 fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes);
419 }
420 request_done(cl);
421 }
422
423 static void handle_uclient_error(struct uclient *cl, int code)
424 {
425 const char *type = "Unknown error";
426 bool ignore = false;
427
428 switch(code) {
429 case UCLIENT_ERROR_CONNECT:
430 type = "Connection failed";
431 error_ret = 4;
432 break;
433 case UCLIENT_ERROR_TIMEDOUT:
434 type = "Connection timed out";
435 error_ret = 4;
436 break;
437 case UCLIENT_ERROR_SSL_INVALID_CERT:
438 type = "Invalid SSL certificate";
439 ignore = !verify;
440 error_ret = 5;
441 break;
442 case UCLIENT_ERROR_SSL_CN_MISMATCH:
443 type = "Server hostname does not match SSL certificate";
444 ignore = !verify;
445 error_ret = 5;
446 break;
447 default:
448 error_ret = 1;
449 break;
450 }
451
452 if (!quiet)
453 fprintf(stderr, "Connection error: %s%s\n", type, ignore ? " (ignored)" : "");
454
455 if (ignore)
456 error_ret = 0;
457 else
458 request_done(cl);
459 }
460
461 static const struct uclient_cb cb = {
462 .header_done = header_done_cb,
463 .data_read = read_data_cb,
464 .data_eof = eof_cb,
465 .error = handle_uclient_error,
466 };
467
468 static int usage(const char *progname)
469 {
470 fprintf(stderr,
471 "Usage: %s [options] <URL>\n"
472 "Options:\n"
473 " -4 Use IPv4 only\n"
474 " -6 Use IPv6 only\n"
475 " -O <file> Redirect output to file (use \"-\" for stdout)\n"
476 " -P <dir> Set directory for output files\n"
477 " --quiet | -q Turn off status messages\n"
478 " --continue | -c Continue a partially-downloaded file\n"
479 " --user=<user> HTTP authentication username\n"
480 " --password=<password> HTTP authentication password\n"
481 " --user-agent | -U <str> Set HTTP user agent\n"
482 " --post-data=STRING use the POST method; send STRING as the data\n"
483 " --post-file=FILE use the POST method; send FILE as the data\n"
484 " --spider | -s Spider mode - only check file existence\n"
485 " --timeout=N | -T N Set connect/request timeout to N seconds\n"
486 " --proxy=on | -Y on Enable interpretation of proxy env vars (default)\n"
487 " --proxy=off | -Y off |\n"
488 " --no-proxy Disable interpretation of proxy env vars\n"
489 "\n"
490 "HTTPS options:\n"
491 " --ca-certificate=<cert> Load CA certificates from file <cert>\n"
492 " --no-check-certificate don't validate the server's certificate\n"
493 " --ciphers=<cipherlist> Set the cipher list string\n"
494 "\n", progname);
495 return 1;
496 }
497
498 static void init_ca_cert(void)
499 {
500 glob_t gl;
501 unsigned int i;
502
503 glob("/etc/ssl/certs/*.crt", 0, NULL, &gl);
504 for (i = 0; i < gl.gl_pathc; i++)
505 ssl_ops->context_add_ca_crt_file(ssl_ctx, gl.gl_pathv[i]);
506 globfree(&gl);
507 }
508
509 static int no_ssl(const char *progname)
510 {
511 fprintf(stderr,
512 "%s: SSL support not available, please install one of the "
513 "libustream-.*[ssl|tls] packages as well as the ca-bundle and "
514 "ca-certificates packages.\n",
515 progname);
516
517 return 1;
518 }
519
520 enum {
521 L_NO_CHECK_CERTIFICATE,
522 L_CA_CERTIFICATE,
523 L_CIPHERS,
524 L_USER,
525 L_PASSWORD,
526 L_USER_AGENT,
527 L_POST_DATA,
528 L_POST_FILE,
529 L_SPIDER,
530 L_TIMEOUT,
531 L_CONTINUE,
532 L_PROXY,
533 L_NO_PROXY,
534 L_QUIET,
535 };
536
537 static const struct option longopts[] = {
538 [L_NO_CHECK_CERTIFICATE] = { "no-check-certificate", no_argument, NULL, 0 },
539 [L_CA_CERTIFICATE] = { "ca-certificate", required_argument, NULL, 0 },
540 [L_CIPHERS] = { "ciphers", required_argument, NULL, 0 },
541 [L_USER] = { "user", required_argument, NULL, 0 },
542 [L_PASSWORD] = { "password", required_argument, NULL, 0 },
543 [L_USER_AGENT] = { "user-agent", required_argument, NULL, 0 },
544 [L_POST_DATA] = { "post-data", required_argument, NULL, 0 },
545 [L_POST_FILE] = { "post-file", required_argument, NULL, 0 },
546 [L_SPIDER] = { "spider", no_argument, NULL, 0 },
547 [L_TIMEOUT] = { "timeout", required_argument, NULL, 0 },
548 [L_CONTINUE] = { "continue", no_argument, NULL, 0 },
549 [L_PROXY] = { "proxy", required_argument, NULL, 0 },
550 [L_NO_PROXY] = { "no-proxy", no_argument, NULL, 0 },
551 [L_QUIET] = { "quiet", no_argument, NULL, 0 },
552 {}
553 };
554
555
556
557 int main(int argc, char **argv)
558 {
559 const char *progname = argv[0];
560 const char *proxy_url;
561 char *username = NULL;
562 char *password = NULL;
563 struct uclient *cl;
564 int longopt_idx = 0;
565 bool has_cert = false;
566 int i, ch;
567 int rc;
568 int af = -1;
569
570 signal(SIGPIPE, SIG_IGN);
571 ssl_ctx = uclient_new_ssl_context(&ssl_ops);
572
573 while ((ch = getopt_long(argc, argv, "46cO:P:qsT:U:Y:", longopts, &longopt_idx)) != -1) {
574 switch(ch) {
575 case 0:
576 switch (longopt_idx) {
577 case L_NO_CHECK_CERTIFICATE:
578 verify = false;
579 if (ssl_ctx)
580 ssl_ops->context_set_require_validation(ssl_ctx, verify);
581 break;
582 case L_CA_CERTIFICATE:
583 has_cert = true;
584 if (ssl_ctx)
585 ssl_ops->context_add_ca_crt_file(ssl_ctx, optarg);
586 break;
587 case L_CIPHERS:
588 if (ssl_ctx) {
589 if (ssl_ops->context_set_ciphers(ssl_ctx, optarg)) {
590 if (!quiet)
591 fprintf(stderr, "No recognized ciphers in cipher list\n");
592 exit(1);
593 }
594 }
595 break;
596 case L_USER:
597 if (!strlen(optarg))
598 break;
599 username = strdupa(optarg);
600 memset(optarg, '*', strlen(optarg));
601 break;
602 case L_PASSWORD:
603 if (!strlen(optarg))
604 break;
605 password = strdupa(optarg);
606 memset(optarg, '*', strlen(optarg));
607 break;
608 case L_USER_AGENT:
609 user_agent = optarg;
610 break;
611 case L_POST_DATA:
612 post_data = optarg;
613 break;
614 case L_POST_FILE:
615 post_file = optarg;
616 break;
617 case L_SPIDER:
618 no_output = true;
619 break;
620 case L_TIMEOUT:
621 timeout = atoi(optarg);
622 break;
623 case L_CONTINUE:
624 resume = true;
625 break;
626 case L_PROXY:
627 if (strcmp(optarg, "on") != 0)
628 proxy = false;
629 break;
630 case L_NO_PROXY:
631 proxy = false;
632 break;
633 case L_QUIET:
634 quiet = true;
635 break;
636 default:
637 return usage(progname);
638 }
639 break;
640 case '4':
641 af = AF_INET;
642 break;
643 case '6':
644 af = AF_INET6;
645 break;
646 case 'c':
647 resume = true;
648 break;
649 case 'U':
650 user_agent = optarg;
651 break;
652 case 'O':
653 opt_output_file = optarg;
654 break;
655 case 'P':
656 if (chdir(optarg)) {
657 if (!quiet)
658 perror("Change output directory");
659 exit(1);
660 }
661 break;
662 case 'q':
663 quiet = true;
664 break;
665 case 's':
666 no_output = true;
667 break;
668 case 'T':
669 timeout = atoi(optarg);
670 break;
671 case 'Y':
672 if (strcmp(optarg, "on") != 0)
673 proxy = false;
674 break;
675 default:
676 return usage(progname);
677 }
678 }
679
680 argv += optind;
681 argc -= optind;
682
683 if (verify && !has_cert)
684 default_certs = true;
685
686 if (argc < 1)
687 return usage(progname);
688
689 if (!ssl_ctx) {
690 for (i = 0; i < argc; i++) {
691 if (!strncmp(argv[i], "https", 5))
692 return no_ssl(progname);
693 }
694 }
695
696 urls = argv + 1;
697 n_urls = argc - 1;
698
699 uloop_init();
700
701 if (username) {
702 if (password) {
703 rc = asprintf(&auth_str, "%s:%s", username, password);
704 if (rc < 0)
705 return rc;
706 } else
707 auth_str = username;
708 }
709
710 if (!quiet)
711 fprintf(stderr, "Downloading '%s'\n", argv[0]);
712
713 proxy_url = get_proxy_url(argv[0]);
714 if (proxy_url) {
715 cl = uclient_new(proxy_url, auth_str, &cb);
716 if (cl)
717 uclient_set_proxy_url(cl, argv[0], NULL);
718 } else {
719 cl = uclient_new(argv[0], auth_str, &cb);
720 }
721 if (!cl) {
722 fprintf(stderr, "Failed to allocate uclient context\n");
723 return 1;
724 }
725 if (af >= 0)
726 uclient_http_set_address_family(cl, af);
727
728 if (ssl_ctx && default_certs)
729 init_ca_cert();
730
731 cur_resume = resume;
732 rc = init_request(cl);
733 if (!rc) {
734 /* no error received, we can enter main loop */
735 uloop_run();
736 } else {
737 fprintf(stderr, "Failed to send request: %s\n", strerror(rc));
738 error_ret = 4;
739 }
740
741 uloop_done();
742
743 uclient_free(cl);
744
745 if (output_fd >= 0 && output_fd != STDOUT_FILENO)
746 close(output_fd);
747
748 if (ssl_ctx)
749 ssl_ops->context_free(ssl_ctx);
750
751 return error_ret;
752 }
libubox HTTP client library