/* * This file is part of the UCB release of Plan 9. It is subject to the license * terms in the LICENSE file found in the top-level directory of this * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No * part of the UCB release of Plan 9, including this file, may be copied, * modified, propagated, or distributed except according to the terms contained * in the LICENSE file. */ #include <u.h>#include <libc.h>#include <ctype.h>#include <bio.h>#include <ip.h>#include <libsec.h>#include <auth.h> typedef struct URL URL;struct URL{ int method; char *host; char *port; char *page; char *etag; char *redirect; char *postbody; char *cred; char *rhead; long mtime;}; typedef struct Range Range;struct Range{ long start; /* only 2 gig supported, tdb */ long end;}; typedef struct Out Out;struct Out{ int fd; int offset; /* notional current offset in output */ int written; /* number of bytes successfully transferred to output */ DigestState *curr; /* digest state up to offset (if known) */ DigestState *hiwat; /* digest state of all bytes written */}; enum{ Other, Http, Https, Ftp,}; enum{ Eof = 0, Error = -1, Server = -2, Changed = -3,}; int debug;char *ofile; int doftp(URL*, URL*, Range*, Out*, long);int dohttp(URL*, URL*, Range*, Out*, long);int crackurl(URL*, char*);Range* crackrange(char*);int getheader(int, char*, int);int httpheaders(int, int, URL*, Range*);int httprcode(int);int cistrncmp(char*, char*, int);int cistrcmp(char*, char*);void initibuf(void);int readline(int, char*, int);int readibuf(int, char*, int);int dfprint(int, char*, ...);void unreadline(char*);int output(Out*, char*, int);void setoffset(Out*, int); int verbose;char *net;char tcpdir[NETPATHLEN];int headerprint; struct { char *name; int (*f)(URL*, URL*, Range*, Out*, long);} method[] = { [Http] { "http", dohttp }, [Https] { "https", dohttp }, [Ftp] { "ftp", doftp }, [Other] { "_______", nil },}; voidusage(void){ fprint(2, "usage: %s [-dhv] [-o outfile] [-p body] [-x netmtpt] [-r header] url\n", argv0); exits("usage");} voidmain(int argc, char **argv){ URL u; Range r; int errs, n; ulong mtime; Dir *d; char postbody[4096], *p, *e, *t, *hpx; URL px; // Proxy Out out; ofile = nil; p = postbody; e = p + sizeof(postbody); r.start = 0; r.end = -1; mtime = 0; memset(&u, 0, sizeof(u)); memset(&px, 0, sizeof(px)); hpx = getenv("httpproxy"); ARGBEGIN { case 'o': ofile = EARGF(usage()); break; case 'd': debug = 1; break; case 'h': headerprint = 1; break; case 'v': verbose = 1; break; case 'x': net = EARGF(usage()); break; case 'r': u.rhead = EARGF(usage()); break; case 'p': t = EARGF(usage()); if(p != postbody) p = seprint(p, e, "&%s", t); else p = seprint(p, e, "%s", t); u.postbody = postbody; break; default: usage(); } ARGEND; if(net != nil){ if(strlen(net) > sizeof(tcpdir)-5) sysfatal("network mount point too long"); snprint(tcpdir, sizeof(tcpdir), "%s/tcp", net); } else snprint(tcpdir, sizeof(tcpdir), "tcp"); if(argc != 1) usage(); out.fd = 1; out.written = 0; out.offset = 0; out.curr = nil; out.hiwat = nil; if(ofile != nil){ d = dirstat(ofile); if(d == nil){ out.fd = create(ofile, OWRITE, 0664); if(out.fd < 0) sysfatal("creating %s: %r", ofile); } else { out.fd = open(ofile, OWRITE); if(out.fd < 0) sysfatal("can't open %s: %r", ofile); r.start = d->length; mtime = d->mtime; free(d); } } errs = 0; if(crackurl(&u, argv[0]) < 0) sysfatal("%r"); if(hpx && crackurl(&px, hpx) < 0) sysfatal("%r"); for(;;){ setoffset(&out, 0); /* transfer data */ werrstr(""); n = (*method[u.method].f)(&u, &px, &r, &out, mtime); switch(n){ case Eof: exits(0); break; case Error: if(errs++ < 10) continue; sysfatal("too many errors with no progress %r"); break; case Server: sysfatal("server returned: %r"); break; } /* forward progress */ errs = 0; r.start += n; if(r.start >= r.end) break; } exits(0);} intcrackurl(URL *u, char *s){ char *p; int i; if(u->page != nil){ free(u->page); u->page = nil; } /* get type */ for(p = s; *p; p++){ if(*p == '/'){ p = s; if(u->method == Other){ werrstr("missing method"); return -1; } if(u->host == nil){ werrstr("missing host"); return -1; } u->page = strdup(p); return 0; } if(*p == ':' && *(p+1)=='/' && *(p+2)=='/'){ *p = 0; p += 3; for(i = 0; i < nelem(method); i++){ if(cistrcmp(s, method[i].name) == 0){ u->method = i; break; } } break; } } if(u->method == Other){ werrstr("unsupported URL type %s", s); return -1; } /* get system */ free(u->host); s = p; p = strchr(s, '/'); if(p == nil){ u->host = strdup(s); u->page = strdup("/"); } else { u->page = strdup(p); *p = 0; u->host = strdup(s); *p = '/'; } if(p = strchr(u->host, ':')) { *p++ = 0; u->port = p; } else u->port = method[u->method].name; if(*(u->host) == 0){ werrstr("bad url, null host"); return -1; } return 0;} char *day[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; char *month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; struct{ int fd; long mtime;} note; voidcatch(void*, char*){ Dir d; nulldir(&d); d.mtime = note.mtime; if(dirfwstat(note.fd, &d) < 0) sysfatal("catch: can't dirfwstat: %r"); noted(NDFLT);} intdohttp(URL *u, URL *px, Range *r, Out *out, long mtime){ int fd, cfd; int redirect, auth, loop; int n, rv, code; long tot, vtime; Tm *tm; char buf[1024]; char err[ERRMAX]; /* always move back to a previous 512 byte bound because some * servers can't seem to deal with requests that start at the * end of the file */ if(r->start) r->start = ((r->start-1)/512)*512; /* loop for redirects, requires reading both response code and headers */ fd = -1; for(loop = 0; loop < 32; loop++){ if(px->host == nil){ fd = dial(netmkaddr(u->host, tcpdir, u->port), 0, 0, 0); } else { fd = dial(netmkaddr(px->host, tcpdir, px->port), 0, 0, 0); } if(fd < 0) return Error; if(u->method == Https){ int tfd; TLSconn conn; memset(&conn, 0, sizeof conn); tfd = tlsClient(fd, &conn); if(tfd < 0){ fprint(2, "tlsClient: %r\n"); close(fd); return Error; } /* BUG: check cert here? */ if(conn.cert) free(conn.cert); close(fd); fd = tfd; } /* write request, use range if not start of file */ if(u->postbody == nil){ if(px->host == nil){ dfprint(fd, "GET %s HTTP/1.0\r\n" "Host: %s\r\n" "User-agent: Plan9/hget\r\n" "Cache-Control: no-cache\r\n" "Pragma: no-cache\r\n", u->page, u->host); } else { dfprint(fd, "GET http://%s%s HTTP/1.0\r\n" "Host: %s\r\n" "User-agent: Plan9/hget\r\n" "Cache-Control: no-cache\r\n" "Pragma: no-cache\r\n", u->host, u->page, u->host); } } else { dfprint(fd, "POST %s HTTP/1.0\r\n" "Host: %s\r\n" "Content-type: application/x-www-form-urlencoded\r\n" "Content-length: %d\r\n" "User-agent: Plan9/hget\r\n", u->page, u->host, strlen(u->postbody)); } if(u->cred) dfprint(fd, "Authorization: Basic %s\r\n", u->cred); if(u->rhead) dfprint(fd, "%s\r\n", u->rhead); if(r->start != 0){ dfprint(fd, "Range: bytes=%d-\n", r->start); if(u->etag != nil){ dfprint(fd, "If-range: %s\n", u->etag); } else { tm = gmtime(mtime); dfprint(fd, "If-range: %s, %d %s %d %2d:%2.2d:%2.2d GMT\n", day[tm->wday], tm->mday, month[tm->mon], tm->year+1900, tm->hour, tm->min, tm->sec); } } if((cfd = open("/mnt/webcookies/http", ORDWR)) >= 0){ if(fprint(cfd, "http://%s%s", u->host, u->page) > 0){ while((n = read(cfd, buf, sizeof buf)) > 0){ if(debug) write(2, buf, n); write(fd, buf, n); } }else{ close(cfd); cfd = -1; } } dfprint(fd, "\r\n", u->host); if(u->postbody) dfprint(fd, "%s", u->postbody); auth = 0; redirect = 0; initibuf(); code = httprcode(fd); switch(code){ case Error: /* connection timed out */ case Eof: close(fd); close(cfd); return code; case 200: /* OK */ case 201: /* Created */ case 202: /* Accepted */ if(ofile == nil && r->start != 0) sysfatal("page changed underfoot"); break; case 204: /* No Content */ sysfatal("No Content"); case 206: /* Partial Content */ setoffset(out, r->start); break; case 301: /* Moved Permanently */ case 302: /* Moved Temporarily (actually Found) */ case 303: /* See Other */ case 307: /* Temporary Redirect (HTTP/1.1) */ redirect = 1; u->postbody = nil; break; case 304: /* Not Modified */ break; case 400: /* Bad Request */ sysfatal("Bad Request"); case 401: /* Unauthorized */ if (auth) sysfatal("Authentication failed"); auth = 1; break; case 402: /* ??? */ sysfatal("Unauthorized"); case 403: /* Forbidden */ sysfatal("Forbidden by server"); case 404: /* Not Found */ sysfatal("Not found on server"); case 407: /* Proxy Authentication */ sysfatal("Proxy authentication required"); case 500: /* Internal server error */ sysfatal("Server choked"); case 501: /* Not implemented */ sysfatal("Server can't do it!"); case 502: /* Bad gateway */ sysfatal("Bad gateway"); Lines 1–500 of 1489. Load more
/* * This file is part of the UCB release of Plan 9. It is subject to the license * terms in the LICENSE file found in the top-level directory of this * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No * part of the UCB release of Plan 9, including this file, may be copied, * modified, propagated, or distributed except according to the terms contained * in the LICENSE file. */ #include <u.h> #include <libc.h> #include <ctype.h> #include <bio.h> #include <ip.h> #include <libsec.h> #include <auth.h> typedef struct URL URL; struct URL { int method; char *host; char *port; char *page; char *etag; char *redirect; char *postbody; char *cred; char *rhead; long mtime; }; typedef struct Range Range; struct Range { long start; /* only 2 gig supported, tdb */ long end; }; typedef struct Out Out; struct Out { int fd; int offset; /* notional current offset in output */ int written; /* number of bytes successfully transferred to output */ DigestState *curr; /* digest state up to offset (if known) */ DigestState *hiwat; /* digest state of all bytes written */ }; enum { Other, Http, Https, Ftp, }; enum { Eof = 0, Error = -1, Server = -2, Changed = -3, }; int debug; char *ofile; int doftp(URL*, URL*, Range*, Out*, long); int dohttp(URL*, URL*, Range*, Out*, long); int crackurl(URL*, char*); Range* crackrange(char*); int getheader(int, char*, int); int httpheaders(int, int, URL*, Range*); int httprcode(int); int cistrncmp(char*, char*, int); int cistrcmp(char*, char*); void initibuf(void); int readline(int, char*, int); int readibuf(int, char*, int); int dfprint(int, char*, ...); void unreadline(char*); int output(Out*, char*, int); void setoffset(Out*, int); int verbose; char *net; char tcpdir[NETPATHLEN]; int headerprint; struct { char *name; int (*f)(URL*, URL*, Range*, Out*, long); } method[] = { [Http] { "http", dohttp }, [Https] { "https", dohttp }, [Ftp] { "ftp", doftp }, [Other] { "_______", nil }, }; void usage(void) { fprint(2, "usage: %s [-dhv] [-o outfile] [-p body] [-x netmtpt] [-r header] url\n", argv0); exits("usage"); } void main(int argc, char **argv) { URL u; Range r; int errs, n; ulong mtime; Dir *d; char postbody[4096], *p, *e, *t, *hpx; URL px; // Proxy Out out; ofile = nil; p = postbody; e = p + sizeof(postbody); r.start = 0; r.end = -1; mtime = 0; memset(&u, 0, sizeof(u)); memset(&px, 0, sizeof(px)); hpx = getenv("httpproxy"); ARGBEGIN { case 'o': ofile = EARGF(usage()); break; case 'd': debug = 1; break; case 'h': headerprint = 1; break; case 'v': verbose = 1; break; case 'x': net = EARGF(usage()); break; case 'r': u.rhead = EARGF(usage()); break; case 'p': t = EARGF(usage()); if(p != postbody) p = seprint(p, e, "&%s", t); else p = seprint(p, e, "%s", t); u.postbody = postbody; break; default: usage(); } ARGEND; if(net != nil){ if(strlen(net) > sizeof(tcpdir)-5) sysfatal("network mount point too long"); snprint(tcpdir, sizeof(tcpdir), "%s/tcp", net); } else snprint(tcpdir, sizeof(tcpdir), "tcp"); if(argc != 1) usage(); out.fd = 1; out.written = 0; out.offset = 0; out.curr = nil; out.hiwat = nil; if(ofile != nil){ d = dirstat(ofile); if(d == nil){ out.fd = create(ofile, OWRITE, 0664); if(out.fd < 0) sysfatal("creating %s: %r", ofile); } else { out.fd = open(ofile, OWRITE); if(out.fd < 0) sysfatal("can't open %s: %r", ofile); r.start = d->length; mtime = d->mtime; free(d); } } errs = 0; if(crackurl(&u, argv[0]) < 0) sysfatal("%r"); if(hpx && crackurl(&px, hpx) < 0) sysfatal("%r"); for(;;){ setoffset(&out, 0); /* transfer data */ werrstr(""); n = (*method[u.method].f)(&u, &px, &r, &out, mtime); switch(n){ case Eof: exits(0); break; case Error: if(errs++ < 10) continue; sysfatal("too many errors with no progress %r"); break; case Server: sysfatal("server returned: %r"); break; } /* forward progress */ errs = 0; r.start += n; if(r.start >= r.end) break; } exits(0); } int crackurl(URL *u, char *s) { char *p; int i; if(u->page != nil){ free(u->page); u->page = nil; } /* get type */ for(p = s; *p; p++){ if(*p == '/'){ p = s; if(u->method == Other){ werrstr("missing method"); return -1; } if(u->host == nil){ werrstr("missing host"); return -1; } u->page = strdup(p); return 0; } if(*p == ':' && *(p+1)=='/' && *(p+2)=='/'){ *p = 0; p += 3; for(i = 0; i < nelem(method); i++){ if(cistrcmp(s, method[i].name) == 0){ u->method = i; break; } } break; } } if(u->method == Other){ werrstr("unsupported URL type %s", s); return -1; } /* get system */ free(u->host); s = p; p = strchr(s, '/'); if(p == nil){ u->host = strdup(s); u->page = strdup("/"); } else { u->page = strdup(p); *p = 0; u->host = strdup(s); *p = '/'; } if(p = strchr(u->host, ':')) { *p++ = 0; u->port = p; } else u->port = method[u->method].name; if(*(u->host) == 0){ werrstr("bad url, null host"); return -1; } return 0; } char *day[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; char *month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; struct { int fd; long mtime; } note; void catch(void*, char*) { Dir d; nulldir(&d); d.mtime = note.mtime; if(dirfwstat(note.fd, &d) < 0) sysfatal("catch: can't dirfwstat: %r"); noted(NDFLT); } int dohttp(URL *u, URL *px, Range *r, Out *out, long mtime) { int fd, cfd; int redirect, auth, loop; int n, rv, code; long tot, vtime; Tm *tm; char buf[1024]; char err[ERRMAX]; /* always move back to a previous 512 byte bound because some * servers can't seem to deal with requests that start at the * end of the file */ if(r->start) r->start = ((r->start-1)/512)*512; /* loop for redirects, requires reading both response code and headers */ fd = -1; for(loop = 0; loop < 32; loop++){ if(px->host == nil){ fd = dial(netmkaddr(u->host, tcpdir, u->port), 0, 0, 0); } else { fd = dial(netmkaddr(px->host, tcpdir, px->port), 0, 0, 0); } if(fd < 0) return Error; if(u->method == Https){ int tfd; TLSconn conn; memset(&conn, 0, sizeof conn); tfd = tlsClient(fd, &conn); if(tfd < 0){ fprint(2, "tlsClient: %r\n"); close(fd); return Error; } /* BUG: check cert here? */ if(conn.cert) free(conn.cert); close(fd); fd = tfd; } /* write request, use range if not start of file */ if(u->postbody == nil){ if(px->host == nil){ dfprint(fd, "GET %s HTTP/1.0\r\n" "Host: %s\r\n" "User-agent: Plan9/hget\r\n" "Cache-Control: no-cache\r\n" "Pragma: no-cache\r\n", u->page, u->host); } else { dfprint(fd, "GET http://%s%s HTTP/1.0\r\n" "Host: %s\r\n" "User-agent: Plan9/hget\r\n" "Cache-Control: no-cache\r\n" "Pragma: no-cache\r\n", u->host, u->page, u->host); } } else { dfprint(fd, "POST %s HTTP/1.0\r\n" "Host: %s\r\n" "Content-type: application/x-www-form-urlencoded\r\n" "Content-length: %d\r\n" "User-agent: Plan9/hget\r\n", u->page, u->host, strlen(u->postbody)); } if(u->cred) dfprint(fd, "Authorization: Basic %s\r\n", u->cred); if(u->rhead) dfprint(fd, "%s\r\n", u->rhead); if(r->start != 0){ dfprint(fd, "Range: bytes=%d-\n", r->start); if(u->etag != nil){ dfprint(fd, "If-range: %s\n", u->etag); } else { tm = gmtime(mtime); dfprint(fd, "If-range: %s, %d %s %d %2d:%2.2d:%2.2d GMT\n", day[tm->wday], tm->mday, month[tm->mon], tm->year+1900, tm->hour, tm->min, tm->sec); } } if((cfd = open("/mnt/webcookies/http", ORDWR)) >= 0){ if(fprint(cfd, "http://%s%s", u->host, u->page) > 0){ while((n = read(cfd, buf, sizeof buf)) > 0){ if(debug) write(2, buf, n); write(fd, buf, n); } }else{ close(cfd); cfd = -1; } } dfprint(fd, "\r\n", u->host); if(u->postbody) dfprint(fd, "%s", u->postbody); auth = 0; redirect = 0; initibuf(); code = httprcode(fd); switch(code){ case Error: /* connection timed out */ case Eof: close(fd); close(cfd); return code; case 200: /* OK */ case 201: /* Created */ case 202: /* Accepted */ if(ofile == nil && r->start != 0) sysfatal("page changed underfoot"); break; case 204: /* No Content */ sysfatal("No Content"); case 206: /* Partial Content */ setoffset(out, r->start); break; case 301: /* Moved Permanently */ case 302: /* Moved Temporarily (actually Found) */ case 303: /* See Other */ case 307: /* Temporary Redirect (HTTP/1.1) */ redirect = 1; u->postbody = nil; break; case 304: /* Not Modified */ break; case 400: /* Bad Request */ sysfatal("Bad Request"); case 401: /* Unauthorized */ if (auth) sysfatal("Authentication failed"); auth = 1; break; case 402: /* ??? */ sysfatal("Unauthorized"); case 403: /* Forbidden */ sysfatal("Forbidden by server"); case 404: /* Not Found */ sysfatal("Not found on server"); case 407: /* Proxy Authentication */ sysfatal("Proxy authentication required"); case 500: /* Internal server error */ sysfatal("Server choked"); case 501: /* Not implemented */ sysfatal("Server can't do it!"); case 502: /* Bad gateway */ sysfatal("Bad gateway");