mdn: update scraper to mdn/yari

Simon Legner committed 5 years ago

Commit 4dc95571 parent 92d8a8e4 files · +18 −12Browse files

Unified · Split

4 files

@@ -41,6 +41,18 @@ module Docs · 40 unmodified lines
41	node.parent['id'] = node['name']	41	node.parent['id'] = node['name']
42	node.before(node.content).remove	42	node.before(node.content).remove
43	end	43	end
		44	css('h2 > a, h3 > a').each do \|node\|
		45	node.parent.content = node.content
		46	end
		47
		48	css('.notecard > h4').each do \|node\|
		49	node.name = 'strong'
		50	end
		51
		52	css('svg.deprecated').each do \|node\|
		53	node.name = 'span'
		54	node.content = node.content
		55	end
44		56
45	css('dt > a[id]').each do \|node\|	57	css('dt > a[id]').each do \|node\|
46	next if node['href']	58	next if node['href']
@@ -64,6 +76,9 @@ module Docs · 17 unmodified lines
64	end	76	end
65		77
66	# New compatibility tables	78	# New compatibility tables
		79	# FIXME(2021):
		80	# - fetched from external JSON: https://developer.mozilla.org/en-US/docs/Web/SVG/Attribute/alignment-baseline/bcd.json
		81	# - https://github.com/mdn/yari/blob/master/build/bcd-urls.js
67		82
68	css('.bc-data #Legend + dl', '.bc-data #Legend', '.bc-data #Legend_2 + dl', '.bc-data #Legend_2', '.bc-browser-name').remove	83	css('.bc-data #Legend + dl', '.bc-data #Legend', '.bc-data #Legend_2 + dl', '.bc-data #Legend_2', '.bc-browser-name').remove
69		84

@@ -11,7 +11,7 @@ module Docs · 10 unmodified lines
11	end	11	end
12		12
13	def other	13	def other
14	css('.prevnext').remove	14	css('.prev-next').remove
15		15
16	if at_css('p').content.include?("\u{00AB}")	16	if at_css('p').content.include?("\u{00AB}")
17	at_css('p').remove	17	at_css('p').remove

     self.abstract = true
     self.type = 'mdn'
-    params[:raw] = 1
-    params[:macros] = 1
     html_filters.push 'mdn/clean_html'
-    options[:rate_limit] = 200
+    options[:container] = '#content'
     options[:trailing_slash] = false
     options[:skip_link] = ->(link) {
     def get_latest_version(opts)
       get_latest_github_commit_date('mdn', 'content', opts)
     end
-    private
-    def process_response?(response)
-      response.effective_url.host = 'developer.mozilla.org' if response.effective_url.host == 'wiki.developer.mozilla.org'
-      super && response.effective_url.query == 'raw=1&macros=1'
-    end
   end
 end