# Webgrab -- for getting html pages and the subordinate files (images, frame children)# they refer to (using "src=..." in a tag) into the local file space.# Assume http: scheme if none specified.# Usage:#	webgrab [-r] [-v] [-o stem] url#  If stem is specified, file will be saved in stem.html and images will#  go in stem_1.jpg (or .gif, ...), stem_2.jpg, etc.#  If stem is not specified, derive it from url (see getstem comment, below).# If -r is specified, get "raw", i.e., no image fetching/html munging.# If -v is specified (verbose), print some progress information,# with more if -vv is given. implement Webgrab; include "sys.m";	sys: Sys;	FD: import sys; include "draw.m"; include "string.m";	S: String; include "url.m";	U: Url;	ParsedUrl: import U; include "daytime.m";	DT: Daytime; include "bufio.m";	B: Bufio; include "dial.m";	D: Dial; include "arg.m"; Webgrab: module{	init: fn(ctxt: ref Draw->Context, args: list of string);}; stderr: ref FD;verbose := 0;postbody : string; httpproxy: ref Url->ParsedUrl;noproxydoms: list of string;	# domains that don't require proxy init(nil: ref Draw->Context, args: list of string){	sys = load Sys Sys->PATH;	stderr = sys->fildes(2);	S = load String String->PATH;	U = load Url Url->PATH;	DT = load Daytime Daytime->PATH;	D = load Dial Dial->PATH;	B = load Bufio Bufio->PATH;	arg := load Arg Arg->PATH;	if(S == nil || U == nil || DT == nil || B == nil || arg == nil)		error_exit("can't load a module");	U->init();	stem := "";	rawflag := 0;	arg->init(args);	arg->setusage("webgrab [-r] [-v[v]] [-p postbody] [-o stem] url");	url := "";	while((o := arg->opt()) != 0)		case o {		'r' =>			rawflag = 1;		'v' =>			verbose++;		'o' =>			stem = arg->earg();		'p' =>			postbody = arg->earg();		* =>			arg->usage();		}	args = arg->argv();	if(len args != 1)		arg->usage();	url = hd args;	arg = nil;	(nil,xr) := S->splitstrl(url,"//");	(nil,yr) := S->splitl(url,":");	if(xr == "" && yr == "")		url = "http://" + url;	u := U->makeurl(url);	if(stem == "")		stem = getstem(u);	readconfig();	grab(u, stem, rawflag);} readconfig(){	cfgio := B->open("/services/webget/config", sys->OREAD);	if(cfgio != nil) {		for(;;) {			line := B->cfgio.gets('\n');			if(line == "") {				B->cfgio.close();				break;			}			if(line[0]=='#')				continue;			(key, val) := S->splitl(line, " \t=");			val = S->take(S->drop(val, " \t="), "^\r\n");			if(val == "")				continue;			case key {			"httpproxy" =>				if(val == "none")					continue;				# val should be host or host:port				httpproxy = U->makeurl("http://" + val);				if(verbose)					sys->fprint(stderr, "Using http proxy %s\n", httpproxy.tostring());			"noproxy" or			"noproxydoms" =>				(nil, noproxydoms) = sys->tokenize(val, ";, \t");			}		}	}} # Make up a stem for forming save-file-names, based on url u.# Use the last non-nil component of u.path, without a final extension,# else use the host.  Then, if the stem still contains a '.' (e.g., www.lucent)# use the part after the final '.'.# Finally, if all else fails, use use "grabout".getstem(u: ref ParsedUrl) : string{	stem := "";	if(u.path != "") {		(l, r) := S->splitr(u.path, "/");		if(r == "") {			# path ended with '/'; try next to last component			if(l != "")				(l, r) = S->splitr(l[0:len l - 1], "/");		}		if(r != "")			stem = r;	}	if(stem == "")		stem = u.host;	if(stem != "") {		ext: string;		(stem, ext) = S->splitr(stem, ".");		if(stem == "")			stem = ext;		else			stem = stem[0:len stem - 1];		(nil, stem) = S->splitr(stem, ".");	}	if(stem == "")		stem = "grabout";	return stem;} grab(u: ref ParsedUrl, stem: string, rawflag: int){	(err, contents, fd, actual) := httpget(u);	if(err != "")		error_exit(err);	ish := is_html(contents);	if(ish)		contents = addfetchcomment(contents, u, actual);	if(rawflag || !ish) {		writebytes(stem, contents, fd);		return;	}	# get subordinates, modify contents	subs : list of (string, string);	(contents, subs)  = subfix(contents, stem);	writebytes(stem + ".html", contents, fd);	for(l := subs; l != nil; l = tl l) {		(fname, suburl) := hd l;		subu := U->makeurl(suburl);		subu.makeabsolute(actual);		(suberr, subcontents, subfd, nil) := httpget(subu);		if(suberr != "") {			sys->fprint(stderr, "webgrab: can't fetch subordinate %s from %s: %s\n", fname, subu.tostring(), suberr);			continue;		}		writebytes(fname, subcontents, subfd);	}} # Fix the html in array a so that referenced subordinate files (SRC= or BACKGROUND= fields of tags)# are replaced with local names (stem_1.xxx, stem_2.xxx, etc.),# and return the fixed array along with a list of (local name, subordinate url)# of images to be fetched.subfix(a: array of byte, stem: string) : (array of byte, list of (string, string)){	alen := len a;	if(alen == 0)		return (a, nil);	nsubs := 0;	newa := array[alen + 1000] of byte;	newai := 0;	j := 0;	intag := 0;	incom := 0;	quote := 0;	subs : list of (string, string) = nil;	for(i := 0; i < alen; i++) {		c := int a[i];		if(incom) {			if(amatch(a, i, alen, "-->")) {				incom = 0;				i = i+2;			}		}		else if(intag) {			if(quote==0 && (amatch(a, i, alen, "src") || amatch(a, i, alen, "background"))) {				v := "";				eqi := 0;				if(amatch(a, i, alen, "src"))					k := i+3;				else					k = i+10;				for(; k < alen; k++)					if(!iswhite(int a[k]))						break;				if(k < alen && int a[k] == '=') {					eqi = k;					k++;					while(k<alen && iswhite(int a[k]))						k++;					if(k<alen) {						kstart := k;						c = int a[k];						if(c == '\'' || c== '"') {							quote = int a[k++];							while(k<alen && (int a[k])!=quote)								k++;							v = string a[kstart+1:k];							k++;						}						else {							while(k<alen && !iswhite(int a[k]) && int a[k] != '>')								k++;							v = string a[kstart:k];						}					}				}				if(v != "") {					f := "";					for(l := subs; l != nil; l = tl l) {						(ff,uu) := hd l;						if(v == uu) {							f = ff;							break;						}					}					if(f == "") {						nsubs++;						f = stem + "_" + string nsubs + getsuff(v);						subs = (f, v) :: subs;					}					# should check for newa too small					newa[newai:] = a[j:eqi+1];					newai += eqi+1-j;					xa := array of byte f;					newa[newai:] = xa;					newai += len xa;					j = k;				}				i = k-1;			}			if(c == '>' && quote == 0)				intag = 0;			if(quote) {				if(quote == c)					quote = 0;			else if(c == '"' || c == '\'')				quote = c;			}		}		else if(c == '<')			intag = 1;	}	if(nsubs == 0)		return (a, nil);	if(i > j) {		newa[newai:] = a[j:i];		newai += i-j;	}	ans := array[newai] of byte;	ans[0:] = newa[0:newai];	anssubs : list of (string, string) = nil;	for(ll := subs; ll != nil; ll = tl ll)		anssubs = hd ll :: anssubs;	return (ans, anssubs);} # add c after all f's in afixnames(a: array of byte, f: string, c: byte){	alen := len a;	n := alen - len f;	for(i := 0; i < n; i++) {		if(amatch(a, i, alen, f)) {			a[i+len f] = c;		}	}} amatch(a: array of byte, i, alen: int, s: string) : int{	slen := len s;	for(k := 0; i+k < alen && k < slen; k++) {		c := int a[i+k];		if(c >= 'A' && c <= 'Z')			c = c + (int 'a' - int 'A');		if(c != s[k])			break;	}	if(k == slen) {		return 1;	}	return 0;} getsuff(ustr: string) : string{	u := U->makeurl(ustr);	if(u.path != "") {		for(i := len u.path - 1; i >= 0; i--) {			c := u.path[i];			if(c == '.')				return u.path[i:];			if(c == '/')				break;		}	}	return "";} iswhite(c: int) : int{	return (c==' ' || c=='\t' || c=='\n' || c=='\r');} # Add a comment to end of a giving date and source of fetchaddfetchcomment(a: array of byte, u, actu: ref ParsedUrl) : array of byte{	now := DT->text(DT->local(DT->now()));	ustr := u.tostring();	actustr := actu.tostring();	comment := "\n<!-- Fetched " + now + " from " + ustr;	if(ustr != actustr)		comment += ", redirected to " + actustr;	comment += " -->\n";	acom := array of byte comment;	newa := array[len a + len acom] of byte;	newa[0:] = a;	newa[len a:] = acom;	return newa;} # Get u, return (error string, body, actual url of source, after redirection)httpget(u: ref ParsedUrl) : (string, array of byte, ref Sys->FD, ref ParsedUrl){	ans, body : array of byte;	restfd: ref Sys->FD;	req : string;		for(redir := 0; redir < 10; redir++) {		if(u.port == "")			u.port = "80";	# default IP port for HTTP		if(verbose)			sys->fprint(stderr, "connecting to %s\n", u.host);		dialhost, port: string; 		if(httpproxy != nil && need_proxy(u.host)) {			dialhost = httpproxy.host;			port = httpproxy.port;		}		else {			dialhost = u.host;			port = u.port;		}		dest := D->netmkaddr(dialhost, "tcp", port);		net := D->dial(dest, nil);		if(net == nil)			return (sys->sprint("can't dial %s: %r", dest), nil, nil, nil);					# prepare request		if(u.query != ""){			u.query = "?" + u.query;		} 		if (postbody == nil){			if(httpproxy == nil || !need_proxy(u.host)){				req = sys->sprint("GET /%s%s HTTP/1.0\r\n"+						"Host: %s\r\n"+						"User-agent: Inferno/webgrab\r\n"+						"Cache-Control: no-cache\r\n"+						"Pragma: no-cache\r\n\r\n",						u.path, u.query, u.host);			}else{				req = sys->sprint("GET http:///%s%s HTTP/1.0\r\n"+						"Host: %s\r\n"+						"User-agent: Inferno/webgrab\r\n"+						"Cache-Control: no-cache\r\n"+						"Pragma: no-cache\r\n\r\n",						u.host, u.path, u.host);			}		}else{				req = sys->sprint("POST /%s HTTP/1.0\r\n"+						"Host: %s\r\n"+						"Content-type: application/x-www-form-urlencoded\r\n"+						"Content-length: %d\r\n"+						"User-agent: Inferno/webgrab\r\n"+						"\r\n"+"%s",						u.path, u.host, len postbody, postbody); 		} 		if(verbose)			sys->fprint(stderr, "writing request: %s\n", req);		areq := array of byte req;		n := sys->write(net.dfd, areq, len areq);		if(n != len areq)			return (sys->sprint("write problem: %r"), nil, nil, nil);		(ans, restfd) = readbytes(net.dfd);		(status, rest) := stripline(ans);		if(verbose)			sys->fprint(stderr, "response: %s\n", status);		(vers, statusrest) := S->splitl(status, " ");		if(!S->prefix("HTTP/", vers))			return ("bad reply status: " + status, rest, restfd, nil);		code := int statusrest;		location := "";		body = rest;		for(;;) {			hline: string;			(hline, body) = stripline(body);			if(hline == "")				break;			if(verbose > 1)				sys->fprint(stderr, "%s\n", hline);			if(!iswhite(hline[0])) {				(hname, hrest) := S->splitl(hline, ":");				if(hrest != "") {					hname = S->tolower(hname);					hval := S->drop(hrest, ": \t");					hval = S->take(hval, "^ \t");					if(hname == "location")						location = hval;				}			}		}		if(code != 200) {			if((code == 300 || code == 301 || code == 302) && location != "") {				# MultipleChoices, MovedPerm, or MovedTemp				if(verbose)					sys->fprint(stderr, "redirect to %s\n", location);				u = U->makeurl(location);				continue; 			}			return ("status not ok: " + status, rest, restfd, u);		}		break;	}	return ("", body, restfd, u);}  need_proxy(h: string) : int{	doml := noproxydoms;	if(doml == nil)		return 1;		# all domains need proxy 	lh := len h;	for(dom := hd doml; doml != nil; doml = tl doml) {		ld := len dom;		if(lh >= ld && h[lh-ld:] == dom)			return 0;	# domain is on the noproxy list	} 	return 1;} # Simple guess test for HTML: first non-white byte is '<'is_html(a: array of byte) : int{	for(i := 0; i < len a; i++)		if(!iswhite(int a[i]))			break;	if(i < len a && a[i] == byte '<')		return 1;	return 0;}

Lines 1–500 of 571. Load more