1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright 2005 Colin Percival
5 * All rights reserved
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted providing that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31
32 #include <sys/types.h>
33 #include <sys/time.h>
34 #include <sys/socket.h>
35
36 #include <ctype.h>
37 #include <err.h>
38 #include <errno.h>
39 #include <fcntl.h>
40 #include <limits.h>
41 #include <netdb.h>
42 #include <stdint.h>
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <string.h>
46 #include <sysexits.h>
47 #include <unistd.h>
48
49 static const char * env_HTTP_PROXY;
50 static char * env_HTTP_PROXY_AUTH;
51 static const char * env_HTTP_USER_AGENT;
52 static char * env_HTTP_TIMEOUT;
53 static const char * proxyport;
54 static char * proxyauth;
55
56 static struct timeval timo = { 15, 0};
57
58 static void
usage(void)59 usage(void)
60 {
61
62 fprintf(stderr, "usage: phttpget server [file ...]\n");
63 exit(EX_USAGE);
64 }
65
66 /*
67 * Base64 encode a string; the string returned, if non-NULL, is
68 * allocated using malloc() and must be freed by the caller.
69 */
70 static char *
b64enc(const char * ptext)71 b64enc(const char *ptext)
72 {
73 static const char base64[] =
74 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
75 "abcdefghijklmnopqrstuvwxyz"
76 "0123456789+/";
77 const char *pt;
78 char *ctext, *pc;
79 size_t ptlen, ctlen;
80 uint32_t t;
81 unsigned int j;
82
83 /*
84 * Encoded length is 4 characters per 3-byte block or partial
85 * block of plaintext, plus one byte for the terminating NUL
86 */
87 ptlen = strlen(ptext);
88 if (ptlen > ((SIZE_MAX - 1) / 4) * 3 - 2)
89 return NULL; /* Possible integer overflow */
90 ctlen = 4 * ((ptlen + 2) / 3) + 1;
91 if ((ctext = malloc(ctlen)) == NULL)
92 return NULL;
93 ctext[ctlen - 1] = 0;
94
95 /*
96 * Scan through ptext, reading up to 3 bytes from ptext and
97 * writing 4 bytes to ctext, until we run out of input.
98 */
99 for (pt = ptext, pc = ctext; ptlen; ptlen -= 3, pc += 4) {
100 /* Read 3 bytes */
101 for (t = j = 0; j < 3; j++) {
102 t <<= 8;
103 if (j < ptlen)
104 t += *pt++;
105 }
106
107 /* Write 4 bytes */
108 for (j = 0; j < 4; j++) {
109 if (j <= ptlen + 1)
110 pc[j] = base64[(t >> 18) & 0x3f];
111 else
112 pc[j] = '=';
113 t <<= 6;
114 }
115
116 /* If we're done, exit the loop */
117 if (ptlen <= 3)
118 break;
119 }
120
121 return (ctext);
122 }
123
124 static void
readenv(void)125 readenv(void)
126 {
127 char *proxy_auth_userpass, *proxy_auth_userpass64, *p;
128 char *proxy_auth_user = NULL;
129 char *proxy_auth_pass = NULL;
130 long http_timeout;
131
132 env_HTTP_PROXY = getenv("HTTP_PROXY");
133 if (env_HTTP_PROXY == NULL)
134 env_HTTP_PROXY = getenv("http_proxy");
135 if (env_HTTP_PROXY != NULL) {
136 if (strncmp(env_HTTP_PROXY, "http://", 7) == 0)
137 env_HTTP_PROXY += 7;
138 p = strchr(env_HTTP_PROXY, '/');
139 if (p != NULL)
140 *p = 0;
141 p = strchr(env_HTTP_PROXY, ':');
142 if (p != NULL) {
143 *p = 0;
144 proxyport = p + 1;
145 } else
146 proxyport = "3128";
147 }
148
149 env_HTTP_PROXY_AUTH = getenv("HTTP_PROXY_AUTH");
150 if ((env_HTTP_PROXY != NULL) &&
151 (env_HTTP_PROXY_AUTH != NULL) &&
152 (strncasecmp(env_HTTP_PROXY_AUTH, "basic:" , 6) == 0)) {
153 /* Ignore authentication scheme */
154 (void) strsep(&env_HTTP_PROXY_AUTH, ":");
155
156 /* Ignore realm */
157 (void) strsep(&env_HTTP_PROXY_AUTH, ":");
158
159 /* Obtain username and password */
160 proxy_auth_user = strsep(&env_HTTP_PROXY_AUTH, ":");
161 proxy_auth_pass = env_HTTP_PROXY_AUTH;
162 }
163
164 if ((proxy_auth_user != NULL) && (proxy_auth_pass != NULL)) {
165 asprintf(&proxy_auth_userpass, "%s:%s",
166 proxy_auth_user, proxy_auth_pass);
167 if (proxy_auth_userpass == NULL)
168 err(1, "asprintf");
169
170 proxy_auth_userpass64 = b64enc(proxy_auth_userpass);
171 if (proxy_auth_userpass64 == NULL)
172 err(1, "malloc");
173
174 asprintf(&proxyauth, "Proxy-Authorization: Basic %s\r\n",
175 proxy_auth_userpass64);
176 if (proxyauth == NULL)
177 err(1, "asprintf");
178
179 free(proxy_auth_userpass);
180 free(proxy_auth_userpass64);
181 } else
182 proxyauth = NULL;
183
184 env_HTTP_USER_AGENT = getenv("HTTP_USER_AGENT");
185 if (env_HTTP_USER_AGENT == NULL)
186 env_HTTP_USER_AGENT = "phttpget/0.1";
187
188 env_HTTP_TIMEOUT = getenv("HTTP_TIMEOUT");
189 if (env_HTTP_TIMEOUT != NULL) {
190 http_timeout = strtol(env_HTTP_TIMEOUT, &p, 10);
191 if ((*env_HTTP_TIMEOUT == '\0') || (*p != '\0') ||
192 (http_timeout < 0))
193 warnx("HTTP_TIMEOUT (%s) is not a positive integer",
194 env_HTTP_TIMEOUT);
195 else
196 timo.tv_sec = http_timeout;
197 }
198 }
199
200 static int
makerequest(char ** buf,char * path,char * server,int connclose)201 makerequest(char ** buf, char * path, char * server, int connclose)
202 {
203 int buflen;
204
205 buflen = asprintf(buf,
206 "GET %s%s/%s HTTP/1.1\r\n"
207 "Host: %s\r\n"
208 "User-Agent: %s\r\n"
209 "%s"
210 "%s"
211 "\r\n",
212 env_HTTP_PROXY ? "http://" : "",
213 env_HTTP_PROXY ? server : "",
214 path, server, env_HTTP_USER_AGENT,
215 proxyauth ? proxyauth : "",
216 connclose ? "Connection: Close\r\n" : "Connection: Keep-Alive\r\n");
217 if (buflen == -1)
218 err(1, "asprintf");
219 return(buflen);
220 }
221
222 static int
readln(int sd,char * resbuf,int * resbuflen,int * resbufpos)223 readln(int sd, char * resbuf, int * resbuflen, int * resbufpos)
224 {
225 ssize_t len;
226
227 while (strnstr(resbuf + *resbufpos, "\r\n",
228 *resbuflen - *resbufpos) == NULL) {
229 /* Move buffered data to the start of the buffer */
230 if (*resbufpos != 0) {
231 memmove(resbuf, resbuf + *resbufpos,
232 *resbuflen - *resbufpos);
233 *resbuflen -= *resbufpos;
234 *resbufpos = 0;
235 }
236
237 /* If the buffer is full, complain */
238 if (*resbuflen == BUFSIZ)
239 return -1;
240
241 /* Read more data into the buffer */
242 len = recv(sd, resbuf + *resbuflen, BUFSIZ - *resbuflen, 0);
243 if ((len == 0) ||
244 ((len == -1) && (errno != EINTR)))
245 return -1;
246
247 if (len != -1)
248 *resbuflen += len;
249 }
250
251 return 0;
252 }
253
254 static int
copybytes(int sd,int fd,off_t copylen,char * resbuf,int * resbuflen,int * resbufpos)255 copybytes(int sd, int fd, off_t copylen, char * resbuf, int * resbuflen,
256 int * resbufpos)
257 {
258 ssize_t len;
259
260 while (copylen) {
261 /* Write data from resbuf to fd */
262 len = *resbuflen - *resbufpos;
263 if (copylen < len)
264 len = copylen;
265 if (len > 0) {
266 if (fd != -1)
267 len = write(fd, resbuf + *resbufpos, len);
268 if (len == -1)
269 err(1, "write");
270 *resbufpos += len;
271 copylen -= len;
272 continue;
273 }
274
275 /* Read more data into buffer */
276 len = recv(sd, resbuf, BUFSIZ, 0);
277 if (len == -1) {
278 if (errno == EINTR)
279 continue;
280 return -1;
281 } else if (len == 0) {
282 return -2;
283 } else {
284 *resbuflen = len;
285 *resbufpos = 0;
286 }
287 }
288
289 return 0;
290 }
291
292 int
main(int argc,char * argv[])293 main(int argc, char *argv[])
294 {
295 struct addrinfo hints; /* Hints to getaddrinfo */
296 struct addrinfo *res; /* Pointer to server address being used */
297 struct addrinfo *res0; /* Pointer to server addresses */
298 char * resbuf = NULL; /* Response buffer */
299 int resbufpos = 0; /* Response buffer position */
300 int resbuflen = 0; /* Response buffer length */
301 char * eolp; /* Pointer to "\r\n" within resbuf */
302 char * hln; /* Pointer within header line */
303 char * servername; /* Name of server */
304 char * fname = NULL; /* Name of downloaded file */
305 char * reqbuf = NULL; /* Request buffer */
306 int reqbufpos = 0; /* Request buffer position */
307 int reqbuflen = 0; /* Request buffer length */
308 ssize_t len; /* Length sent or received */
309 int nreq = 0; /* Number of next request to send */
310 int nres = 0; /* Number of next reply to receive */
311 int pipelined = 0; /* != 0 if connection in pipelined mode. */
312 int keepalive; /* != 0 if HTTP/1.0 keep-alive rcvd. */
313 int sd = -1; /* Socket descriptor */
314 int sdflags = 0; /* Flags on the socket sd */
315 int fd = -1; /* Descriptor for download target file */
316 int error; /* Error code */
317 int statuscode; /* HTTP Status code */
318 off_t contentlength; /* Value from Content-Length header */
319 int chunked; /* != if transfer-encoding is chunked */
320 off_t clen; /* Chunk length */
321 int firstreq = 0; /* # of first request for this connection */
322 int val; /* Value used for setsockopt call */
323
324 /* Check that the arguments are sensible */
325 if (argc < 2)
326 usage();
327
328 /* Read important environment variables */
329 readenv();
330
331 /* Get server name and adjust arg[cv] to point at file names */
332 servername = argv[1];
333 argv += 2;
334 argc -= 2;
335
336 /* Allocate response buffer */
337 resbuf = malloc(BUFSIZ);
338 if (resbuf == NULL)
339 err(1, "malloc");
340
341 /* Look up server */
342 memset(&hints, 0, sizeof(hints));
343 hints.ai_family = PF_UNSPEC;
344 hints.ai_socktype = SOCK_STREAM;
345 error = getaddrinfo(env_HTTP_PROXY ? env_HTTP_PROXY : servername,
346 env_HTTP_PROXY ? proxyport : "http", &hints, &res0);
347 if (error)
348 errx(1, "host = %s, port = %s: %s",
349 env_HTTP_PROXY ? env_HTTP_PROXY : servername,
350 env_HTTP_PROXY ? proxyport : "http",
351 gai_strerror(error));
352 if (res0 == NULL)
353 errx(1, "could not look up %s", servername);
354 res = res0;
355
356 /* Do the fetching */
357 while (nres < argc) {
358 /* Make sure we have a connected socket */
359 for (; sd == -1; res = res->ai_next) {
360 /* No addresses left to try :-( */
361 if (res == NULL)
362 errx(1, "Could not connect to %s", servername);
363
364 /* Create a socket... */
365 sd = socket(res->ai_family, res->ai_socktype,
366 res->ai_protocol);
367 if (sd == -1)
368 continue;
369
370 /* ... set 15-second timeouts ... */
371 setsockopt(sd, SOL_SOCKET, SO_SNDTIMEO,
372 (void *)&timo, (socklen_t)sizeof(timo));
373 setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO,
374 (void *)&timo, (socklen_t)sizeof(timo));
375
376 /* ... disable SIGPIPE generation ... */
377 val = 1;
378 setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE,
379 (void *)&val, sizeof(int));
380
381 /* ... and connect to the server. */
382 if(connect(sd, res->ai_addr, res->ai_addrlen)) {
383 close(sd);
384 sd = -1;
385 continue;
386 }
387
388 firstreq = nres;
389 }
390
391 /*
392 * If in pipelined HTTP mode, put socket into non-blocking
393 * mode, since we're probably going to want to try to send
394 * several HTTP requests.
395 */
396 if (pipelined) {
397 sdflags = fcntl(sd, F_GETFL);
398 if (fcntl(sd, F_SETFL, sdflags | O_NONBLOCK) == -1)
399 err(1, "fcntl");
400 }
401
402 /* Construct requests and/or send them without blocking */
403 while ((nreq < argc) && ((reqbuf == NULL) || pipelined)) {
404 /* If not in the middle of a request, make one */
405 if (reqbuf == NULL) {
406 reqbuflen = makerequest(&reqbuf, argv[nreq],
407 servername, (nreq == argc - 1));
408 reqbufpos = 0;
409 }
410
411 /* If in pipelined mode, try to send the request */
412 if (pipelined) {
413 while (reqbufpos < reqbuflen) {
414 len = send(sd, reqbuf + reqbufpos,
415 reqbuflen - reqbufpos, 0);
416 if (len == -1)
417 break;
418 reqbufpos += len;
419 }
420 if (reqbufpos < reqbuflen) {
421 if (errno != EAGAIN)
422 goto conndied;
423 break;
424 } else {
425 free(reqbuf);
426 reqbuf = NULL;
427 nreq++;
428 }
429 }
430 }
431
432 /* Put connection back into blocking mode */
433 if (pipelined) {
434 if (fcntl(sd, F_SETFL, sdflags) == -1)
435 err(1, "fcntl");
436 }
437
438 /* Do we need to blocking-send a request? */
439 if (nres == nreq) {
440 while (reqbufpos < reqbuflen) {
441 len = send(sd, reqbuf + reqbufpos,
442 reqbuflen - reqbufpos, 0);
443 if (len == -1)
444 goto conndied;
445 reqbufpos += len;
446 }
447 free(reqbuf);
448 reqbuf = NULL;
449 nreq++;
450 }
451
452 /* Scan through the response processing headers. */
453 statuscode = 0;
454 contentlength = -1;
455 chunked = 0;
456 keepalive = 0;
457 do {
458 /* Get a header line */
459 error = readln(sd, resbuf, &resbuflen, &resbufpos);
460 if (error)
461 goto conndied;
462 hln = resbuf + resbufpos;
463 eolp = strnstr(hln, "\r\n", resbuflen - resbufpos);
464 resbufpos = (eolp - resbuf) + 2;
465 *eolp = '\0';
466
467 /* Make sure it doesn't contain a NUL character */
468 if (strchr(hln, '\0') != eolp)
469 goto conndied;
470
471 if (statuscode == 0) {
472 /* The first line MUST be HTTP/1.x xxx ... */
473 if ((strncmp(hln, "HTTP/1.", 7) != 0) ||
474 ! isdigit(hln[7]))
475 goto conndied;
476
477 /*
478 * If the minor version number isn't zero,
479 * then we can assume that pipelining our
480 * requests is OK -- as long as we don't
481 * see a "Connection: close" line later
482 * and we either have a Content-Length or
483 * Transfer-Encoding: chunked header to
484 * tell us the length.
485 */
486 if (hln[7] != '0')
487 pipelined = 1;
488
489 /* Skip over the minor version number */
490 hln = strchr(hln + 7, ' ');
491 if (hln == NULL)
492 goto conndied;
493 else
494 hln++;
495
496 /* Read the status code */
497 while (isdigit(*hln)) {
498 statuscode = statuscode * 10 +
499 *hln - '0';
500 hln++;
501 }
502
503 if (statuscode < 100 || statuscode > 599)
504 goto conndied;
505
506 /* Ignore the rest of the line */
507 continue;
508 }
509
510 /*
511 * Check for "Connection: close" or
512 * "Connection: Keep-Alive" header
513 */
514 if (strncasecmp(hln, "Connection:", 11) == 0) {
515 hln += 11;
516 if (strcasestr(hln, "close") != NULL)
517 pipelined = 0;
518 if (strcasestr(hln, "Keep-Alive") != NULL)
519 keepalive = 1;
520
521 /* Next header... */
522 continue;
523 }
524
525 /* Check for "Content-Length:" header */
526 if (strncasecmp(hln, "Content-Length:", 15) == 0) {
527 hln += 15;
528 contentlength = 0;
529
530 /* Find the start of the length */
531 while (!isdigit(*hln) && (*hln != '\0'))
532 hln++;
533
534 /* Compute the length */
535 while (isdigit(*hln)) {
536 if (contentlength >= OFF_MAX / 10) {
537 /* Nasty people... */
538 goto conndied;
539 }
540 contentlength = contentlength * 10 +
541 *hln - '0';
542 hln++;
543 }
544
545 /* Next header... */
546 continue;
547 }
548
549 /* Check for "Transfer-Encoding: chunked" header */
550 if (strncasecmp(hln, "Transfer-Encoding:", 18) == 0) {
551 hln += 18;
552 if (strcasestr(hln, "chunked") != NULL)
553 chunked = 1;
554
555 /* Next header... */
556 continue;
557 }
558
559 /* We blithely ignore any other header lines */
560
561 /* No more header lines */
562 if (strlen(hln) == 0) {
563 /*
564 * If the status code was 1xx, then there will
565 * be a real header later. Servers may emit
566 * 1xx header blocks at will, but since we
567 * don't expect one, we should just ignore it.
568 */
569 if (100 <= statuscode && statuscode <= 199) {
570 statuscode = 0;
571 continue;
572 }
573
574 /* End of header; message body follows */
575 break;
576 }
577 } while (1);
578
579 /* No message body for 204 or 304 */
580 if (statuscode == 204 || statuscode == 304) {
581 nres++;
582 continue;
583 }
584
585 /*
586 * There should be a message body coming, but we only want
587 * to send it to a file if the status code is 200
588 */
589 if (statuscode == 200) {
590 /* Generate a file name for the download */
591 fname = strrchr(argv[nres], '/');
592 if (fname == NULL)
593 fname = argv[nres];
594 else
595 fname++;
596 if (strlen(fname) == 0)
597 errx(1, "Cannot obtain file name from %s\n",
598 argv[nres]);
599
600 fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
601 if (fd == -1)
602 errx(1, "open(%s)", fname);
603 }
604
605 /* Read the message and send data to fd if appropriate */
606 if (chunked) {
607 /* Handle a chunked-encoded entity */
608
609 /* Read chunks */
610 do {
611 error = readln(sd, resbuf, &resbuflen,
612 &resbufpos);
613 if (error)
614 goto conndied;
615 hln = resbuf + resbufpos;
616 eolp = strstr(hln, "\r\n");
617 resbufpos = (eolp - resbuf) + 2;
618
619 clen = 0;
620 while (isxdigit(*hln)) {
621 if (clen >= OFF_MAX / 16) {
622 /* Nasty people... */
623 goto conndied;
624 }
625 if (isdigit(*hln))
626 clen = clen * 16 + *hln - '0';
627 else
628 clen = clen * 16 + 10 +
629 tolower(*hln) - 'a';
630 hln++;
631 }
632
633 error = copybytes(sd, fd, clen, resbuf,
634 &resbuflen, &resbufpos);
635 if (error) {
636 goto conndied;
637 }
638 } while (clen != 0);
639
640 /* Read trailer and final CRLF */
641 do {
642 error = readln(sd, resbuf, &resbuflen,
643 &resbufpos);
644 if (error)
645 goto conndied;
646 hln = resbuf + resbufpos;
647 eolp = strstr(hln, "\r\n");
648 resbufpos = (eolp - resbuf) + 2;
649 } while (hln != eolp);
650 } else if (contentlength != -1) {
651 error = copybytes(sd, fd, contentlength, resbuf,
652 &resbuflen, &resbufpos);
653 if (error)
654 goto conndied;
655 } else {
656 /*
657 * Not chunked, and no content length header.
658 * Read everything until the server closes the
659 * socket.
660 */
661 error = copybytes(sd, fd, OFF_MAX, resbuf,
662 &resbuflen, &resbufpos);
663 if (error == -1)
664 goto conndied;
665 pipelined = 0;
666 }
667
668 if (fd != -1) {
669 close(fd);
670 fd = -1;
671 }
672
673 fprintf(stderr, "http://%s/%s: %d ", servername, argv[nres],
674 statuscode);
675 if (statuscode == 200)
676 fprintf(stderr, "OK\n");
677 else if (statuscode < 300)
678 fprintf(stderr, "Successful (ignored)\n");
679 else if (statuscode < 400)
680 fprintf(stderr, "Redirection (ignored)\n");
681 else
682 fprintf(stderr, "Error (ignored)\n");
683
684 /* We've finished this file! */
685 nres++;
686
687 /*
688 * If necessary, clean up this connection so that we
689 * can start a new one.
690 */
691 if (pipelined == 0 && keepalive == 0)
692 goto cleanupconn;
693 continue;
694
695 conndied:
696 /*
697 * Something went wrong -- our connection died, the server
698 * sent us garbage, etc. If this happened on the first
699 * request we sent over this connection, give up. Otherwise,
700 * close this connection, open a new one, and reissue the
701 * request.
702 */
703 if (nres == firstreq)
704 errx(1, "Connection failure");
705
706 cleanupconn:
707 /*
708 * Clean up our connection and keep on going
709 */
710 shutdown(sd, SHUT_RDWR);
711 close(sd);
712 sd = -1;
713 if (fd != -1) {
714 close(fd);
715 fd = -1;
716 }
717 if (reqbuf != NULL) {
718 free(reqbuf);
719 reqbuf = NULL;
720 }
721 nreq = nres;
722 res = res0;
723 pipelined = 0;
724 resbufpos = resbuflen = 0;
725 continue;
726 }
727
728 free(resbuf);
729 freeaddrinfo(res0);
730
731 return 0;
732 }
733