mdn: update scraper to mdn/yari

Simon Legner committed 5 years ago

Commit 4dc95571 parent 92d8a8e4 files · +18 −12Browse files

Unified · Split

4 files

@@ -41,6 +41,18 @@ module Docs · 40 unmodified lines
41	41	node.parent['id'] = node['name']
42	42	node.before(node.content).remove
43	43	end
	44	+ css('h2 > a, h3 > a').each do \|node\|
	45	+ node.parent.content = node.content
	46	+ end
	47	+
	48	+ css('.notecard > h4').each do \|node\|
	49	+ node.name = 'strong'
	50	+ end
	51	+
	52	+ css('svg.deprecated').each do \|node\|
	53	+ node.name = 'span'
	54	+ node.content = node.content
	55	+ end
44	56
45	57	css('dt > a[id]').each do \|node\|
46	58	next if node['href']
@@ -64,6 +76,9 @@ module Docs · 17 unmodified lines
64	76	end
65	77
66	78	# New compatibility tables
	79	+ # FIXME(2021):
	80	+ # - fetched from external JSON: https://developer.mozilla.org/en-US/docs/Web/SVG/Attribute/alignment-baseline/bcd.json
	81	+ # - https://github.com/mdn/yari/blob/master/build/bcd-urls.js
67	82
68	83	css('.bc-data #Legend + dl', '.bc-data #Legend', '.bc-data #Legend_2 + dl', '.bc-data #Legend_2', '.bc-browser-name').remove
69	84

@@ -11,7 +11,7 @@ module Docs · 10 unmodified lines
11	11	end
12	12
13	13	def other
14		− css('.prevnext').remove
	14	+ css('.prev-next').remove
15	15
16	16	if at_css('p').content.include?("\u{00AB}")
17	17	at_css('p').remove

@@ -3,12 +3,9 @@ module Docs · 2 unmodified lines
3	3	self.abstract = true
4	4	self.type = 'mdn'
5	5
6		− params[:raw] = 1
7		− params[:macros] = 1
8		−
9	6	html_filters.push 'mdn/clean_html'
10	7
11		− options[:rate_limit] = 200
	8	+ options[:container] = '#content'
12	9	options[:trailing_slash] = false
13	10
14	11	options[:skip_link] = ->(link) {
@@ -23,12 +20,5 @@ module Docs · 8 unmodified lines
23	20	def get_latest_version(opts)
24	21	get_latest_github_commit_date('mdn', 'content', opts)
25	22	end
26		−
27		− private
28		−
29		− def process_response?(response)
30		− response.effective_url.host = 'developer.mozilla.org' if response.effective_url.host == 'wiki.developer.mozilla.org'
31		− super && response.effective_url.query == 'raw=1&macros=1'
32		− end
33	23	end
34	24	end