Change where GitHub scraper gets its JSON. Fix Koa, Q, Sanctuary, and Lua/nginx imports

spamguy committed 1 year ago

Commit 3794bf61 parent b661f689 files · +30 −20Browse files

Unified · Split

9 files

@@ -2,6 +2,11 @@ module Docs · 1 unmodified line
2	2	class Github
3	3	class CleanHtmlFilter < Filter
4	4	def call
	5	+ # Remove h1 wrapper to render it correctly.
	6	+ css('.markdown-heading h1').each do \|node\|
	7	+ node.parent.replace(node)
	8	+ end
	9	+
5	10	css('.anchor').each do \|node\|
6	11	node.parent['id'] = node['href'].remove('#')
7	12	node.remove

@@ -4,11 +4,11 @@ module Docs · 3 unmodified lines
4	4	def additional_entries
5	5	entries = []
6	6
7		− css('#directives + ul > li > a').each do \|node\|
	7	+ css('h2:contains("Directives") + ul > li > a').each do \|node\|
8	8	entries << [node.content, node['href'].remove('#'), 'Directives']
9	9	end
10	10
11		− css('#nginx-api-for-lua + ul > li > a').each do \|node\|
	11	+ css('h2:contains("Nginx API for Lua") + ul > li > a').each do \|node\|
12	12	next if node.content == 'Introduction'
13	13	entries << [node.content, node['href'].remove('#'), 'Nginx API for Lua']
14	14	end

@@ -39,8 +39,10 @@ module Docs · 38 unmodified lines
39	39	when "h3"
40	40	type = node.text
41	41	when "h4"
	42	+ # Parent <div>'s ID set in github/clean_html.
	43	+ id = node.parent.attributes["id"].value
42	44	name = node.text.split(' :: ')[0]
43		− id = node.attributes["id"].value
	45	+
44	46	entries << [name, id, type]
45	47	end
46	48	end

@@ -8,11 +8,6 @@ module Docs · 7 unmodified lines
8	8	node.name = 'h3'
9	9	}
10	10
11		− # correct and unify link ids
12		− css('h3').each { \|node\|
13		− node.attributes["id"].value = node.text.split(' :: ')[0]
14		− }
15		−
16	11	doc
17	12	end
18	13	end

@@ -38,9 +38,9 @@ module Docs · 37 unmodified lines
38	38	case node.name
39	39	when "h2"
40	40	type = node.text
41		− if node.attributes["id"].value == "type-class-hierarchy"
	41	+ if node.parent.attributes["id"]&.value == "type-class-hierarchy"
42	42	name = node.text
43		− id = node.attributes["id"].value
	43	+ id = node.parent.attributes["id"].value
44	44	entries << [name, id, type]
45	45	end
46	46	when "h4"

@@ -16,7 +16,13 @@ module Docs · 15 unmodified lines
16	16	end
17	17
18	18	def parse(response)
19		− parsed = JSON.parse(response.response_body)
	19	+ embedded_json = response
	20	+ .response_body
	21	+ .match(/react-app\.embeddedData">(.+?)<\/script>/)
	22	+ &.captures
	23	+ &.first
	24	+ parsed = JSON.parse(embedded_json)
	25	+
20	26	[parsed['payload']['blob']['richText'], parsed['title']]
21	27	end
22	28	end

@@ -2,7 +2,7 @@ · 1 unmodified line
2	2
3	3	module Docs
4	4	class Koa < Github
5		− self.base_url = 'https://github.com/koajs/koa/tree/master/docs'
	5	+ self.base_url = 'https://github.com/koajs/koa/blob/master/docs'
6	6	self.release = '2.15.0'
7	7
8	8	self.root_path = 'api/index.md'
@@ -26,7 +26,7 @@ module Docs · 17 unmodified lines
26	26
27	27	html_filters.push 'koa/clean_html', 'koa/entries'
28	28
29		− options[:skip] = %w[middleware.gif]
	29	+ options[:skip_patterns] = [/\.gif/]
30	30	options[:trailing_slash] = false
31	31	options[:container] = '.markdown-body'
32	32

@@ -2,8 +2,9 @@ module Docs · 1 unmodified line
2	2	class NginxLuaModule < Github
3	3	self.name = 'nginx / Lua Module'
4	4	self.slug = 'nginx_lua_module'
5		− self.release = '0.10.13'
6		− self.base_url = "https://github.com/openresty/lua-nginx-module/tree/v#{self.release}/"
	5	+ self.release = '0.10.28'
	6	+ self.base_url = "https://github.com/openresty/lua-nginx-module/blob/v#{self.release}/"
	7	+ self.root_path = 'README.markdown'
7	8	self.links = {
8	9	code: 'https://github.com/openresty/lua-nginx-module'
9	10	}
@@ -11,13 +12,14 @@ module Docs · 1 unmodified line
11	12	html_filters.push 'nginx_lua_module/clean_html', 'nginx_lua_module/entries', 'title'
12	13
13	14	options[:root_title] = 'ngx_http_lua_module'
14		− options[:container] = '#readme > article'
15		−
	15	+ options[:container] = '.markdown-body'
	16	+ options[:max_image_size] = 256_000
16	17	options[:attribution] = <<-HTML
17	18	© 2009–2017 Xiaozhe Wang (chaoslawful)<br>
18		− © 2009–2018 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.<br>
	19	+ © 2009–2019 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.<br>
19	20	Licensed under the BSD License.
20	21	HTML
	22	+ options[:skip_patterns] = [/\.png/]
21	23
22	24	def get_latest_version(opts)
23	25	tags = get_github_tags('openresty', 'lua-nginx-module', opts)

@@ -1,5 +1,5 @@
1	1	module Docs
2		− class Q < Github
	2	+ class Q < UrlScraper
3	3	self.name = 'Q'
4	4	self.release = '1.5.1'
5	5	self.base_url = 'https://github.com/kriskowal/q/wiki/'
@@ -16,7 +16,7 @@ module Docs · 10 unmodified lines
16	16	options[:skip_links] = true
17	17
18	18	options[:attribution] = <<-HTML
19		− © 2009–2017 Kristopher Michael Kowal<br>
	19	+ © 2009–2018 Kristopher Michael Kowal<br>
20	20	Licensed under the MIT License.
21	21	HTML
22	22