Unified · Split
| @@ -40,6 +40,7 @@ Webgrab: module · 39 unmodified lines | ||
| 40 | 40 | |
| 41 | 41 | stderr: ref FD; |
| 42 | 42 | verbose := 0; |
| 43 | +postbody : string; | |
| 43 | 44 | |
| 44 | 45 | httpproxy: ref Url->ParsedUrl; |
| 45 | 46 | noproxydoms: list of string; # domains that don't require proxy |
| @@ -59,7 +60,7 @@ init(nil: ref Draw->Context, args: list of string) · 13 unmodified lines | ||
| 59 | 60 | stem := ""; |
| 60 | 61 | rawflag := 0; |
| 61 | 62 | arg->init(args); |
| 62 | − arg->setusage("webgrab [-r] [-v[v]] [-o stem] url"); | |
| 63 | + arg->setusage("webgrab [-r] [-v[v]] [-p postbody] [-o stem] url"); | |
| 63 | 64 | url := ""; |
| 64 | 65 | while((o := arg->opt()) != 0) |
| 65 | 66 | case o { |
| @@ -69,6 +70,8 @@ init(nil: ref Draw->Context, args: list of string) · 3 unmodified lines | ||
| 69 | 70 | verbose++; |
| 70 | 71 | 'o' => |
| 71 | 72 | stem = arg->earg(); |
| 73 | + 'p' => | |
| 74 | + postbody = arg->earg(); | |
| 72 | 75 | * => |
| 73 | 76 | arg->usage(); |
| 74 | 77 | } |
| @@ -174,7 +177,7 @@ grab(u: ref ParsedUrl, stem: string, rawflag: int) · 99 unmodified lines | ||
| 174 | 177 | (fname, suburl) := hd l; |
| 175 | 178 | subu := U->makeurl(suburl); |
| 176 | 179 | subu.makeabsolute(actual); |
| 177 | − (suberr, subcontents, subfd, subactual) := httpget(subu); | |
| 180 | + (suberr, subcontents, subfd, nil) := httpget(subu); | |
| 178 | 181 | if(suberr != "") { |
| 179 | 182 | sys->fprint(stderr, "webgrab: can't fetch subordinate %s from %s: %s\n", fname, subu.tostring(), suberr); |
| 180 | 183 | continue; |
| @@ -361,17 +364,18 @@ httpget(u: ref ParsedUrl) : (string, array of byte, ref Sys->FD, ref ParsedUrl) · 180 unmodified lines | ||
| 361 | 364 | { |
| 362 | 365 | ans, body : array of byte; |
| 363 | 366 | restfd: ref Sys->FD; |
| 367 | + req : string; | |
| 368 | + | |
| 364 | 369 | for(redir := 0; redir < 10; redir++) { |
| 365 | 370 | if(u.port == "") |
| 366 | 371 | u.port = "80"; # default IP port for HTTP |
| 367 | 372 | if(verbose) |
| 368 | 373 | sys->fprint(stderr, "connecting to %s\n", u.host); |
| 369 | 374 | dialhost, port: string; |
| 370 | − req := "GET "; | |
| 375 | + | |
| 371 | 376 | if(httpproxy != nil && need_proxy(u.host)) { |
| 372 | 377 | dialhost = httpproxy.host; |
| 373 | 378 | port = httpproxy.port; |
| 374 | − req += "http://" + u.host; | |
| 375 | 379 | } |
| 376 | 380 | else { |
| 377 | 381 | dialhost = u.host; |
| @@ -380,10 +384,39 @@ httpget(u: ref ParsedUrl) : (string, array of byte, ref Sys->FD, ref ParsedUrl) · 2 unmodified lines | ||
| 380 | 384 | (ok, net) := sys->dial("tcp!" + dialhost + "!" + port, nil); |
| 381 | 385 | if(ok < 0) |
| 382 | 386 | return (sys->sprint("can't dial %s: %r", dialhost), nil, nil, nil); |
| 383 | − req += "/" + u.path; | |
| 384 | − if(u.query != "") | |
| 385 | − req += "?" + u.query; | |
| 386 | − req += " HTTP/1.0\r\nHost: "+u.host+"\r\nUser-agent: Inferno/webgrab\r\n\r\n"; | |
| 387 | + | |
| 388 | + # prepare request | |
| 389 | + if(u.query != ""){ | |
| 390 | + u.query = "?" + u.query; | |
| 391 | + } | |
| 392 | + | |
| 393 | + if (postbody == nil){ | |
| 394 | + if(httpproxy == nil || !need_proxy(u.host)){ | |
| 395 | + req = sys->sprint("GET /%s%s HTTP/1.0\r\n"+ | |
| 396 | + "Host: %s\r\n"+ | |
| 397 | + "User-agent: Inferno/webgrab\r\n"+ | |
| 398 | + "Cache-Control: no-cache\r\n"+ | |
| 399 | + "Pragma: no-cache\r\n\r\n", | |
| 400 | + u.path, u.query, u.host); | |
| 401 | + }else{ | |
| 402 | + req = sys->sprint("GET http:///%s%s HTTP/1.0\r\n"+ | |
| 403 | + "Host: %s\r\n"+ | |
| 404 | + "User-agent: Inferno/webgrab\r\n"+ | |
| 405 | + "Cache-Control: no-cache\r\n"+ | |
| 406 | + "Pragma: no-cache\r\n\r\n", | |
| 407 | + u.host, u.path, u.host); | |
| 408 | + } | |
| 409 | + }else{ | |
| 410 | + req = sys->sprint("POST /%s HTTP/1.0\r\n"+ | |
| 411 | + "Host: %s\r\n"+ | |
| 412 | + "Content-type: application/x-www-form-urlencoded\r\n"+ | |
| 413 | + "Content-length: %d\r\n"+ | |
| 414 | + "User-agent: Inferno/webgrab\r\n"+ | |
| 415 | + "\r\n"+"%s", | |
| 416 | + u.path, u.host, len postbody, postbody); | |
| 417 | + | |
| 418 | + } | |
| 419 | + | |
| 387 | 420 | if(verbose) |
| 388 | 421 | sys->fprint(stderr, "writing request: %s\n", req); |
| 389 | 422 | areq := array of byte req; |
| @@ -433,6 +466,7 @@ httpget(u: ref ParsedUrl) : (string, array of byte, ref Sys->FD, ref ParsedUrl) · 43 unmodified lines | ||
| 433 | 466 | return ("", body, restfd, u); |
| 434 | 467 | } |
| 435 | 468 | |
| 469 | + | |
| 436 | 470 | need_proxy(h: string) : int |
| 437 | 471 | { |
| 438 | 472 | doml := noproxydoms; |
Binary file not shown.
| @@ -9,6 +9,8 @@ webgrab \- fetch web page content as files · 8 unmodified lines | ||
| 9 | 9 | .B -v |
| 10 | 10 | ] [ |
| 11 | 11 | .BI -o " stem" |
| 12 | +] [ | |
| 13 | +.BI -p " body" | |
| 12 | 14 | ] |
| 13 | 15 | .I url |
| 14 | 16 | .SH DESCRIPTION |
| @@ -93,6 +95,15 @@ print a chatty progress report · 78 unmodified lines | ||
| 93 | 95 | use the |
| 94 | 96 | .I stem |
| 95 | 97 | as given |
| 98 | +.TP | |
| 99 | +.BI -p " body" | |
| 100 | +Use HTTP | |
| 101 | +.B POST | |
| 102 | +instead of | |
| 103 | +.BR GET , | |
| 104 | +posting | |
| 105 | +.I body | |
| 106 | +as the data | |
| 96 | 107 | .PP |
| 97 | 108 | .I Webgrab |
| 98 | 109 | reads the |