20070312c webgrab

Charles.Forsyth committed 19 years ago

Commit e2187311 parent 50734c83 files · +52 −7Browse files

Unified · Split

3 files

appl/cmd/webgrab.b +42 −8
dis/webgrab.dis +0 −0
man/1/webgrab +11 −0

@@ -40,6 +40,7 @@ Webgrab: module · 39 unmodified lines
40		40
41	stderr: ref FD;	41	stderr: ref FD;
42	verbose := 0;	42	verbose := 0;
		43	postbody : string;
43		44
44	httpproxy: ref Url->ParsedUrl;	45	httpproxy: ref Url->ParsedUrl;
45	noproxydoms: list of string; # domains that don't require proxy	46	noproxydoms: list of string; # domains that don't require proxy
@@ -59,7 +60,7 @@ init(nil: ref Draw->Context, args: list of string) · 13 unmodified lines
59	stem := "";	60	stem := "";
60	rawflag := 0;	61	rawflag := 0;
61	arg->init(args);	62	arg->init(args);
62	arg->setusage("webgrab [-r] [-v[v]] [-o stem] url");	63	arg->setusage("webgrab [-r] [-v[v]] [-p postbody] [-o stem] url");
63	url := "";	64	url := "";
64	while((o := arg->opt()) != 0)	65	while((o := arg->opt()) != 0)
65	case o {	66	case o {
@@ -69,6 +70,8 @@ init(nil: ref Draw->Context, args: list of string) · 3 unmodified lines
69	verbose++;	70	verbose++;
70	'o' =>	71	'o' =>
71	stem = arg->earg();	72	stem = arg->earg();
		73	'p' =>
		74	postbody = arg->earg();
72	* =>	75	* =>
73	arg->usage();	76	arg->usage();
74	}	77	}
@@ -174,7 +177,7 @@ grab(u: ref ParsedUrl, stem: string, rawflag: int) · 99 unmodified lines
174	(fname, suburl) := hd l;	177	(fname, suburl) := hd l;
175	subu := U->makeurl(suburl);	178	subu := U->makeurl(suburl);
176	subu.makeabsolute(actual);	179	subu.makeabsolute(actual);
177	(suberr, subcontents, subfd, subactual) := httpget(subu);	180	(suberr, subcontents, subfd, nil) := httpget(subu);
178	if(suberr != "") {	181	if(suberr != "") {
179	sys->fprint(stderr, "webgrab: can't fetch subordinate %s from %s: %s\n", fname, subu.tostring(), suberr);	182	sys->fprint(stderr, "webgrab: can't fetch subordinate %s from %s: %s\n", fname, subu.tostring(), suberr);
180	continue;	183	continue;
@@ -361,17 +364,18 @@ httpget(u: ref ParsedUrl) : (string, array of byte, ref Sys->FD, ref ParsedUrl) · 180 unmodified lines
361	{	364	{
362	ans, body : array of byte;	365	ans, body : array of byte;
363	restfd: ref Sys->FD;	366	restfd: ref Sys->FD;
		367	req : string;
		368
364	for(redir := 0; redir < 10; redir++) {	369	for(redir := 0; redir < 10; redir++) {
365	if(u.port == "")	370	if(u.port == "")
366	u.port = "80"; # default IP port for HTTP	371	u.port = "80"; # default IP port for HTTP
367	if(verbose)	372	if(verbose)
368	sys->fprint(stderr, "connecting to %s\n", u.host);	373	sys->fprint(stderr, "connecting to %s\n", u.host);
369	dialhost, port: string;	374	dialhost, port: string;
370	req := "GET ";	375
371	if(httpproxy != nil && need_proxy(u.host)) {	376	if(httpproxy != nil && need_proxy(u.host)) {
372	dialhost = httpproxy.host;	377	dialhost = httpproxy.host;
373	port = httpproxy.port;	378	port = httpproxy.port;
374	req += "http://" + u.host;
375	}	379	}
376	else {	380	else {
377	dialhost = u.host;	381	dialhost = u.host;
@@ -380,10 +384,39 @@ httpget(u: ref ParsedUrl) : (string, array of byte, ref Sys->FD, ref ParsedUrl) · 2 unmodified lines
380	(ok, net) := sys->dial("tcp!" + dialhost + "!" + port, nil);	384	(ok, net) := sys->dial("tcp!" + dialhost + "!" + port, nil);
381	if(ok < 0)	385	if(ok < 0)
382	return (sys->sprint("can't dial %s: %r", dialhost), nil, nil, nil);	386	return (sys->sprint("can't dial %s: %r", dialhost), nil, nil, nil);
383	req += "/" + u.path;	387
384	if(u.query != "")	388	# prepare request
385	req += "?" + u.query;	389	if(u.query != ""){
386	req += " HTTP/1.0\r\nHost: "+u.host+"\r\nUser-agent: Inferno/webgrab\r\n\r\n";	390	u.query = "?" + u.query;
		391	}
		392
		393	if (postbody == nil){
		394	if(httpproxy == nil \|\| !need_proxy(u.host)){
		395	req = sys->sprint("GET /%s%s HTTP/1.0\r\n"+
		396	"Host: %s\r\n"+
		397	"User-agent: Inferno/webgrab\r\n"+
		398	"Cache-Control: no-cache\r\n"+
		399	"Pragma: no-cache\r\n\r\n",
		400	u.path, u.query, u.host);
		401	}else{
		402	req = sys->sprint("GET http:///%s%s HTTP/1.0\r\n"+
		403	"Host: %s\r\n"+
		404	"User-agent: Inferno/webgrab\r\n"+
		405	"Cache-Control: no-cache\r\n"+
		406	"Pragma: no-cache\r\n\r\n",
		407	u.host, u.path, u.host);
		408	}
		409	}else{
		410	req = sys->sprint("POST /%s HTTP/1.0\r\n"+
		411	"Host: %s\r\n"+
		412	"Content-type: application/x-www-form-urlencoded\r\n"+
		413	"Content-length: %d\r\n"+
		414	"User-agent: Inferno/webgrab\r\n"+
		415	"\r\n"+"%s",
		416	u.path, u.host, len postbody, postbody);
		417
		418	}
		419
387	if(verbose)	420	if(verbose)
388	sys->fprint(stderr, "writing request: %s\n", req);	421	sys->fprint(stderr, "writing request: %s\n", req);
389	areq := array of byte req;	422	areq := array of byte req;
@@ -433,6 +466,7 @@ httpget(u: ref ParsedUrl) : (string, array of byte, ref Sys->FD, ref ParsedUrl) · 43 unmodified lines
433	return ("", body, restfd, u);	466	return ("", body, restfd, u);
434	}	467	}
435		468
		469
436	need_proxy(h: string) : int	470	need_proxy(h: string) : int
437	{	471	{
438	doml := noproxydoms;	472	doml := noproxydoms;

dis/webgrab.dis+0−0

Binary file not shown.

man/1/webgrab+11−0

@@ -9,6 +9,8 @@ webgrab \- fetch web page content as files · 8 unmodified lines
9	.B -v	9	.B -v
10	] [	10	] [
11	.BI -o " stem"	11	.BI -o " stem"
		12	] [
		13	.BI -p " body"
12	]	14	]
13	.I url	15	.I url
14	.SH DESCRIPTION	16	.SH DESCRIPTION
@@ -93,6 +95,15 @@ print a chatty progress report · 78 unmodified lines
93	use the	95	use the
94	.I stem	96	.I stem
95	as given	97	as given
		98	.TP
		99	.BI -p " body"
		100	Use HTTP
		101	.B POST
		102	instead of
		103	.BR GET ,
		104	posting
		105	.I body
		106	as the data
96	.PP	107	.PP
97	.I Webgrab	108	.I Webgrab
98	reads the	109	reads the