uclient-fetch.c

   1 /*
   2  * uclient - ustream based protocol client library
   3  *
   4  * Copyright (C) 2014 Felix Fietkau <nbd@openwrt.org>
   5  *
   6  * Permission to use, copy, modify, and/or distribute this software for any
   7  * purpose with or without fee is hereby granted, provided that the above
   8  * copyright notice and this permission notice appear in all copies.
   9  *
  10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  */
  18
  19 #define _GNU_SOURCE
  20 #include <sys/stat.h>
  21 #include <sys/socket.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24 #include <dlfcn.h>
  25 #include <getopt.h>
  26 #include <fcntl.h>
  27 #include <glob.h>
  28 #include <stdint.h>
  29 #include <inttypes.h>
  30 #include <signal.h>
  31
  32 #include <libubox/blobmsg.h>
  33
  34 #include "progress.h"
  35 #include "uclient.h"
  36 #include "uclient-utils.h"
  37
  38 #ifdef __APPLE__
  39 #define LIB_EXT "dylib"
  40 #else
  41 #define LIB_EXT "so"
  42 #endif
  43
  44 static const char *user_agent = "uclient-fetch";
  45 static const char *post_data;
  46 static struct ustream_ssl_ctx *ssl_ctx;
  47 static const struct ustream_ssl_ops *ssl_ops;
  48 static int quiet = false;
  49 static bool verify = true;
  50 static bool proxy = true;
  51 static bool default_certs = false;
  52 static bool no_output;
  53 static const char *opt_output_file;
  54 static int output_fd = -1;
  55 static int error_ret;
  56 static off_t out_offset;
  57 static off_t out_bytes;
  58 static off_t out_len;
  59 static char *auth_str;
  60 static char **urls;
  61 static int n_urls;
  62 static int timeout;
  63 static bool resume, cur_resume;
  64
  65 static struct progress pmt;
  66 static struct uloop_timeout pmt_timer;
  67
  68 static int init_request(struct uclient *cl);
  69 static void request_done(struct uclient *cl);
  70
  71 static void pmt_update(struct uloop_timeout *t)
  72 {
  73         progress_update(&pmt, out_offset, out_bytes, out_len);
  74         uloop_timeout_set(t, 1000);
  75 }
  76
  77 static const char *
  78 get_proxy_url(char *url)
  79 {
  80         char prefix[16];
  81         char *sep;
  82
  83         if (!proxy)
  84                 return NULL;
  85
  86         sep = strchr(url, ':');
  87         if (!sep)
  88                 return NULL;
  89
  90         if (sep - url > 5)
  91                 return NULL;
  92
  93         memcpy(prefix, url, sep - url);
  94         strcpy(prefix + (sep - url), "_proxy");
  95         return getenv(prefix);
  96 }
  97
  98 static int open_output_file(const char *path, uint64_t resume_offset)
  99 {
 100         const char *output_file = opt_output_file;
 101         char *filename = NULL;
 102         int flags;
 103         int ret;
 104
 105         if (cur_resume)
 106                 flags = O_RDWR;
 107         else
 108                 flags = O_WRONLY | O_TRUNC;
 109
 110         if (!cur_resume && !output_file)
 111                 flags |= O_EXCL;
 112
 113         flags |= O_CREAT;
 114
 115         if (output_file) {
 116                 if (!strcmp(output_file, "-")) {
 117                         if (!quiet)
 118                                 fprintf(stderr, "Writing to stdout\n");
 119
 120                         ret = STDOUT_FILENO;
 121                         goto done;
 122                 }
 123         } else {
 124                 filename = uclient_get_url_filename(path, "index.html");
 125                 if (!filename) {
 126                         ret = -ENOMEM;
 127                         goto out;
 128                 }
 129
 130                 output_file = filename;
 131         }
 132
 133         if (!quiet)
 134                 fprintf(stderr, "Writing to '%s'\n", output_file);
 135         ret = open(output_file, flags, 0644);
 136         if (ret < 0)
 137                 goto free;
 138
 139         if (resume_offset &&
 140             lseek(ret, resume_offset, SEEK_SET) < 0) {
 141                 if (!quiet)
 142                         fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset);
 143                 close(ret);
 144                 ret = -1;
 145                 goto free;
 146         }
 147
 148         out_offset = resume_offset;
 149         out_bytes += resume_offset;
 150 done:
 151         if (!quiet) {
 152                 progress_init(&pmt, output_file);
 153                 pmt_timer.cb = pmt_update;
 154                 pmt_timer.cb(&pmt_timer);
 155         }
 156
 157 free:
 158         free(filename);
 159 out:
 160         return ret;
 161 }
 162
 163 static void header_done_cb(struct uclient *cl)
 164 {
 165         enum {
 166                 H_RANGE,
 167                 H_LEN,
 168                 __H_MAX
 169         };
 170         static const struct blobmsg_policy policy[__H_MAX] = {
 171                 [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING },
 172                 [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING },
 173         };
 174         struct blob_attr *tb[__H_MAX];
 175         uint64_t resume_offset = 0, resume_end, resume_size;
 176         static int retries;
 177
 178         if (retries < 10) {
 179                 int ret = uclient_http_redirect(cl);
 180                 if (ret < 0) {
 181                         if (!quiet)
 182                                 fprintf(stderr, "Failed to redirect to %s on %s\n", cl->url->location, cl->url->host);
 183                         error_ret = 8;
 184                         request_done(cl);
 185                         return;
 186                 }
 187                 if (ret > 0) {
 188                         if (!quiet)
 189                                 fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host);
 190
 191                         retries++;
 192                         return;
 193                 }
 194         }
 195
 196         if (cl->status_code == 204 && cur_resume) {
 197                 /* Resume attempt failed, try normal download */
 198                 cur_resume = false;
 199                 init_request(cl);
 200                 return;
 201         }
 202
 203         blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta));
 204
 205         switch (cl->status_code) {
 206         case 416:
 207                 if (!quiet)
 208                         fprintf(stderr, "File download already fully retrieved; nothing to do.\n");
 209                 request_done(cl);
 210                 break;
 211         case 206:
 212                 if (!cur_resume) {
 213                         if (!quiet)
 214                                 fprintf(stderr, "Error: Partial content received, full content requested\n");
 215                         error_ret = 8;
 216                         request_done(cl);
 217                         break;
 218                 }
 219
 220                 if (!tb[H_RANGE]) {
 221                         if (!quiet)
 222                                 fprintf(stderr, "Content-Range header is missing\n");
 223                         error_ret = 8;
 224                         break;
 225                 }
 226
 227                 if (sscanf(blobmsg_get_string(tb[H_RANGE]),
 228                            "bytes %"PRIu64"-%"PRIu64"/%"PRIu64,
 229                            &resume_offset, &resume_end, &resume_size) != 3) {
 230                         if (!quiet)
 231                                 fprintf(stderr, "Content-Range header is invalid\n");
 232                         error_ret = 8;
 233                         break;
 234                 }
 235         case 204:
 236         case 200:
 237                 if (no_output)
 238                         break;
 239
 240                 if (tb[H_LEN])
 241                         out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10);
 242
 243                 output_fd = open_output_file(cl->url->location, resume_offset);
 244                 if (output_fd < 0) {
 245                         if (!quiet)
 246                                 perror("Cannot open output file");
 247                         error_ret = 3;
 248                         request_done(cl);
 249                 }
 250                 break;
 251
 252         default:
 253                 if (!quiet)
 254                         fprintf(stderr, "HTTP error %d\n", cl->status_code);
 255                 request_done(cl);
 256                 error_ret = 8;
 257                 break;
 258         }
 259 }
 260
 261 static void read_data_cb(struct uclient *cl)
 262 {
 263         char buf[256];
 264         ssize_t n;
 265         int len;
 266
 267         if (!no_output && output_fd < 0)
 268                 return;
 269
 270         while (1) {
 271                 len = uclient_read(cl, buf, sizeof(buf));
 272                 if (len <= 0)
 273                         return;
 274
 275                 out_bytes += len;
 276                 if (!no_output) {
 277                         n = write(output_fd, buf, len);
 278                         if (n < 0)
 279                                 return;
 280                 }
 281         }
 282 }
 283
 284 static void msg_connecting(struct uclient *cl)
 285 {
 286         char addr[INET6_ADDRSTRLEN];
 287         int port;
 288
 289         if (quiet)
 290                 return;
 291
 292         uclient_get_addr(addr, &port, &cl->remote_addr);
 293         fprintf(stderr, "Connecting to %s:%d\n", addr, port);
 294 }
 295
 296 static void check_resume_offset(struct uclient *cl)
 297 {
 298         char range_str[64];
 299         struct stat st;
 300         char *file;
 301         int ret;
 302
 303         file = uclient_get_url_filename(cl->url->location, "index.html");
 304         if (!file)
 305                 return;
 306
 307         ret = stat(file, &st);
 308         free(file);
 309         if (ret)
 310                 return;
 311
 312         if (!st.st_size)
 313                 return;
 314
 315         snprintf(range_str, sizeof(range_str), "bytes=%"PRIu64"-", (uint64_t) st.st_size);
 316         uclient_http_set_header(cl, "Range", range_str);
 317 }
 318
 319 static int init_request(struct uclient *cl)
 320 {
 321         int rc;
 322
 323         out_offset = 0;
 324         out_bytes = 0;
 325         out_len = 0;
 326         uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify);
 327
 328         if (timeout)
 329                 cl->timeout_msecs = timeout * 1000;
 330
 331         rc = uclient_connect(cl);
 332         if (rc)
 333                 return rc;
 334
 335         msg_connecting(cl);
 336
 337         rc = uclient_http_set_request_type(cl, post_data ? "POST" : "GET");
 338         if (rc)
 339                 return rc;
 340
 341         uclient_http_reset_headers(cl);
 342         uclient_http_set_header(cl, "User-Agent", user_agent);
 343         if (cur_resume)
 344                 check_resume_offset(cl);
 345
 346         if (post_data) {
 347                 uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded");
 348                 uclient_write(cl, post_data, strlen(post_data));
 349         }
 350
 351         rc = uclient_request(cl);
 352         if (rc)
 353                 return rc;
 354
 355         return 0;
 356 }
 357
 358 static void request_done(struct uclient *cl)
 359 {
 360         const char *proxy_url;
 361
 362         if (n_urls) {
 363                 proxy_url = get_proxy_url(*urls);
 364                 if (proxy_url) {
 365                         uclient_set_url(cl, proxy_url, NULL);
 366                         uclient_set_proxy_url(cl, *urls, auth_str);
 367                 } else {
 368                         uclient_set_url(cl, *urls, auth_str);
 369                 }
 370                 n_urls--;
 371                 cur_resume = resume;
 372                 error_ret = init_request(cl);
 373                 if (error_ret == 0)
 374                         return;
 375         }
 376
 377         if (output_fd >= 0 && !opt_output_file) {
 378                 close(output_fd);
 379                 output_fd = -1;
 380         }
 381         uclient_disconnect(cl);
 382         uloop_end();
 383 }
 384
 385
 386 static void eof_cb(struct uclient *cl)
 387 {
 388         if (!quiet) {
 389                 pmt_update(&pmt_timer);
 390                 uloop_timeout_cancel(&pmt_timer);
 391                 fprintf(stderr, "\n");
 392         }
 393
 394         if (!cl->data_eof) {
 395                 if (!quiet)
 396                         fprintf(stderr, "Connection reset prematurely\n");
 397                 error_ret = 4;
 398         } else if (!quiet) {
 399                 fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes);
 400         }
 401         request_done(cl);
 402 }
 403
 404 static void handle_uclient_error(struct uclient *cl, int code)
 405 {
 406         const char *type = "Unknown error";
 407         bool ignore = false;
 408
 409         switch(code) {
 410         case UCLIENT_ERROR_CONNECT:
 411                 type = "Connection failed";
 412                 error_ret = 4;
 413                 break;
 414         case UCLIENT_ERROR_TIMEDOUT:
 415                 type = "Connection timed out";
 416                 error_ret = 4;
 417                 break;
 418         case UCLIENT_ERROR_SSL_INVALID_CERT:
 419                 type = "Invalid SSL certificate";
 420                 ignore = !verify;
 421                 error_ret = 5;
 422                 break;
 423         case UCLIENT_ERROR_SSL_CN_MISMATCH:
 424                 type = "Server hostname does not match SSL certificate";
 425                 ignore = !verify;
 426                 error_ret = 5;
 427                 break;
 428         default:
 429                 error_ret = 1;
 430                 break;
 431         }
 432
 433         if (!quiet)
 434                 fprintf(stderr, "Connection error: %s%s\n", type, ignore ? " (ignored)" : "");
 435
 436         if (ignore)
 437                 error_ret = 0;
 438         else
 439                 request_done(cl);
 440 }
 441
 442 static const struct uclient_cb cb = {
 443         .header_done = header_done_cb,
 444         .data_read = read_data_cb,
 445         .data_eof = eof_cb,
 446         .error = handle_uclient_error,
 447 };
 448
 449 static int usage(const char *progname)
 450 {
 451         fprintf(stderr,
 452                 "Usage: %s [options] <URL>\n"
 453                 "Options:\n"
 454                 "       -4                              Use IPv4 only\n"
 455                 "       -6                              Use IPv6 only\n"
 456                 "       -q                              Turn off status messages\n"
 457                 "       -O <file>                       Redirect output to file (use \"-\" for stdout)\n"
 458                 "       -P <dir>                        Set directory for output files\n"
 459                 "       --user=<user>                   HTTP authentication username\n"
 460                 "       --password=<password>           HTTP authentication password\n"
 461                 "       --user-agent|-U <str>           Set HTTP user agent\n"
 462                 "       --post-data=STRING              use the POST method; send STRING as the data\n"
 463                 "       --spider|-s                     Spider mode - only check file existence\n"
 464                 "       --timeout=N|-T N                Set connect/request timeout to N seconds\n"
 465                 "       --proxy=on|off|-Y on|off        Enable/disable env var configured proxy\n"
 466                 "\n"
 467                 "HTTPS options:\n"
 468                 "       --ca-certificate=<cert>         Load CA certificates from file <cert>\n"
 469                 "       --no-check-certificate          don't validate the server's certificate\n"
 470                 "       --ciphers=<cipherlist>          Set the cipher list string\n"
 471                 "\n", progname);
 472         return 1;
 473 }
 474
 475 static void init_ca_cert(void)
 476 {
 477         glob_t gl;
 478         int i;
 479
 480         glob("/etc/ssl/certs/*.crt", 0, NULL, &gl);
 481         for (i = 0; i < gl.gl_pathc; i++)
 482                 ssl_ops->context_add_ca_crt_file(ssl_ctx, gl.gl_pathv[i]);
 483 }
 484
 485 static void init_ustream_ssl(void)
 486 {
 487         void *dlh;
 488
 489         dlh = dlopen("libustream-ssl." LIB_EXT, RTLD_LAZY | RTLD_LOCAL);
 490         if (!dlh)
 491                 return;
 492
 493         ssl_ops = dlsym(dlh, "ustream_ssl_ops");
 494         if (!ssl_ops)
 495                 return;
 496
 497         ssl_ctx = ssl_ops->context_new(false);
 498 }
 499
 500 static int no_ssl(const char *progname)
 501 {
 502         fprintf(stderr,
 503                 "%s: SSL support not available, please install one of the "
 504                 "libustream-.*[ssl|tls] packages as well as the ca-bundle and "
 505                 "ca-certificates packages.\n",
 506                 progname);
 507
 508         return 1;
 509 }
 510
 511 enum {
 512         L_NO_CHECK_CERTIFICATE,
 513         L_CA_CERTIFICATE,
 514         L_CIPHERS,
 515         L_USER,
 516         L_PASSWORD,
 517         L_USER_AGENT,
 518         L_POST_DATA,
 519         L_SPIDER,
 520         L_TIMEOUT,
 521         L_CONTINUE,
 522         L_PROXY,
 523         L_NO_PROXY,
 524         L_QUIET,
 525 };
 526
 527 static const struct option longopts[] = {
 528         [L_NO_CHECK_CERTIFICATE] = { "no-check-certificate", no_argument },
 529         [L_CA_CERTIFICATE] = { "ca-certificate", required_argument },
 530         [L_CIPHERS] = { "ciphers", required_argument },
 531         [L_USER] = { "user", required_argument },
 532         [L_PASSWORD] = { "password", required_argument },
 533         [L_USER_AGENT] = { "user-agent", required_argument },
 534         [L_POST_DATA] = { "post-data", required_argument },
 535         [L_SPIDER] = { "spider", no_argument },
 536         [L_TIMEOUT] = { "timeout", required_argument },
 537         [L_CONTINUE] = { "continue", no_argument },
 538         [L_PROXY] = { "proxy", required_argument },
 539         [L_NO_PROXY] = { "no-proxy", no_argument },
 540         [L_QUIET] = { "quiet", no_argument },
 541         {}
 542 };
 543
 544
 545
 546 int main(int argc, char **argv)
 547 {
 548         const char *progname = argv[0];
 549         const char *proxy_url;
 550         char *username = NULL;
 551         char *password = NULL;
 552         struct uclient *cl;
 553         int longopt_idx = 0;
 554         bool has_cert = false;
 555         int i, ch;
 556         int rc;
 557         int af = -1;
 558
 559         signal(SIGPIPE, SIG_IGN);
 560         init_ustream_ssl();
 561
 562         while ((ch = getopt_long(argc, argv, "46cO:P:qsT:U:Y:", longopts, &longopt_idx)) != -1) {
 563                 switch(ch) {
 564                 case 0:
 565                         switch (longopt_idx) {
 566                         case L_NO_CHECK_CERTIFICATE:
 567                                 verify = false;
 568                                 break;
 569                         case L_CA_CERTIFICATE:
 570                                 has_cert = true;
 571                                 if (ssl_ctx)
 572                                         ssl_ops->context_add_ca_crt_file(ssl_ctx, optarg);
 573                                 break;
 574                         case L_CIPHERS:
 575                                 if (ssl_ctx) {
 576                                         if (ssl_ops->context_set_ciphers(ssl_ctx, optarg)) {
 577                                                 if (!quiet)
 578                                                         fprintf(stderr, "No recognized ciphers in cipher list\n");
 579                                                 exit(1);
 580                                         }
 581                                 }
 582                                 break;
 583                         case L_USER:
 584                                 if (!strlen(optarg))
 585                                         break;
 586                                 username = strdup(optarg);
 587                                 memset(optarg, '*', strlen(optarg));
 588                                 break;
 589                         case L_PASSWORD:
 590                                 if (!strlen(optarg))
 591                                         break;
 592                                 password = strdup(optarg);
 593                                 memset(optarg, '*', strlen(optarg));
 594                                 break;
 595                         case L_USER_AGENT:
 596                                 user_agent = optarg;
 597                                 break;
 598                         case L_POST_DATA:
 599                                 post_data = optarg;
 600                                 break;
 601                         case L_SPIDER:
 602                                 no_output = true;
 603                                 break;
 604                         case L_TIMEOUT:
 605                                 timeout = atoi(optarg);
 606                                 break;
 607                         case L_CONTINUE:
 608                                 resume = true;
 609                                 break;
 610                         case L_PROXY:
 611                                 if (strcmp(optarg, "on") != 0)
 612                                         proxy = false;
 613                                 break;
 614                         case L_NO_PROXY:
 615                                 proxy = false;
 616                                 break;
 617                         case L_QUIET:
 618                                 quiet = true;
 619                                 break;
 620                         default:
 621                                 return usage(progname);
 622                         }
 623                         break;
 624                 case '4':
 625                         af = AF_INET;
 626                         break;
 627                 case '6':
 628                         af = AF_INET6;
 629                         break;
 630                 case 'c':
 631                         resume = true;
 632                         break;
 633                 case 'U':
 634                         user_agent = optarg;
 635                         break;
 636                 case 'O':
 637                         opt_output_file = optarg;
 638                         break;
 639                 case 'P':
 640                         if (chdir(optarg)) {
 641                                 if (!quiet)
 642                                         perror("Change output directory");
 643                                 exit(1);
 644                         }
 645                         break;
 646                 case 'q':
 647                         quiet = true;
 648                         break;
 649                 case 's':
 650                         no_output = true;
 651                         break;
 652                 case 'T':
 653                         timeout = atoi(optarg);
 654                         break;
 655                 case 'Y':
 656                         if (strcmp(optarg, "on") != 0)
 657                                 proxy = false;
 658                         break;
 659                 default:
 660                         return usage(progname);
 661                 }
 662         }
 663
 664         argv += optind;
 665         argc -= optind;
 666
 667         if (verify && !has_cert)
 668                 default_certs = true;
 669
 670         if (argc < 1)
 671                 return usage(progname);
 672
 673         if (!ssl_ctx) {
 674                 for (i = 0; i < argc; i++) {
 675                         if (!strncmp(argv[i], "https", 5))
 676                                 return no_ssl(progname);
 677                 }
 678         }
 679
 680         urls = argv + 1;
 681         n_urls = argc - 1;
 682
 683         uloop_init();
 684
 685         if (username) {
 686                 if (password) {
 687                         rc = asprintf(&auth_str, "%s:%s", username, password);
 688                         if (rc < 0)
 689                                 return rc;
 690                 } else
 691                         auth_str = username;
 692         }
 693
 694         if (!quiet)
 695                 fprintf(stderr, "Downloading '%s'\n", argv[0]);
 696
 697         proxy_url = get_proxy_url(argv[0]);
 698         if (proxy_url) {
 699                 cl = uclient_new(proxy_url, auth_str, &cb);
 700                 if (cl)
 701                     uclient_set_proxy_url(cl, argv[0], NULL);
 702         } else {
 703                 cl = uclient_new(argv[0], auth_str, &cb);
 704         }
 705         if (!cl) {
 706                 fprintf(stderr, "Failed to allocate uclient context\n");
 707                 return 1;
 708         }
 709         if (af >= 0)
 710             uclient_http_set_address_family(cl, af);
 711
 712         if (ssl_ctx && default_certs)
 713                 init_ca_cert();
 714
 715         cur_resume = resume;
 716         rc = init_request(cl);
 717         if (!rc) {
 718                 /* no error received, we can enter main loop */
 719                 uloop_run();
 720         } else {
 721                 fprintf(stderr, "Failed to establish connection\n");
 722                 error_ret = 4;
 723         }
 724
 725         uloop_done();
 726
 727         uclient_free(cl);
 728
 729         if (output_fd >= 0 && output_fd != STDOUT_FILENO)
 730                 close(output_fd);
 731
 732         if (ssl_ctx)
 733                 ssl_ops->context_free(ssl_ctx);
 734
 735         return error_ret;
 736 }