History

Change where GitHub scraper gets its JSON. Fix Koa, Q, Sanctuary, and Lua/nginx imports

spamguy committed 1 year ago

Commit 3794bf61 parent b661f689 files · +30 −20Browse files

Unified · Split

9 files
  1. lib/docs/filters/github/clean_html.rb +5 −0
  2. lib/docs/filters/nginx_lua_module/entries.rb +2 −2
  3. lib/docs/filters/sanctuary_def/entries.rb +3 −1
  4. lib/docs/filters/sanctuary_type_classes/clean_html.rb +0 −5
  5. lib/docs/filters/sanctuary_type_classes/entries.rb +2 −2
  6. lib/docs/scrapers/github.rb +7 −1
  7. lib/docs/scrapers/koa.rb +2 −2
  8. lib/docs/scrapers/nginx_lua_module.rb +7 −5
  9. lib/docs/scrapers/q.rb +2 −2
lib/docs/filters/github/clean_html.rb+5−0
@@ -2,6 +2,11 @@ module Docs · 1 unmodified line
2 class Github2 class Github
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 # Remove h1 wrapper to render it correctly.
6 css('.markdown-heading h1').each do |node|
7 node.parent.replace(node)
8 end
9
5 css('.anchor').each do |node|10 css('.anchor').each do |node|
6 node.parent['id'] = node['href'].remove('#')11 node.parent['id'] = node['href'].remove('#')
7 node.remove12 node.remove
lib/docs/filters/nginx_lua_module/entries.rb+2−2
@@ -4,11 +4,11 @@ module Docs · 3 unmodified lines
4 def additional_entries4 def additional_entries
5 entries = []5 entries = []
6 6
7 css('#directives + ul > li > a').each do |node|7 css('h2:contains("Directives") + ul > li > a').each do |node|
8 entries << [node.content, node['href'].remove('#'), 'Directives']8 entries << [node.content, node['href'].remove('#'), 'Directives']
9 end9 end
10 10
11 css('#nginx-api-for-lua + ul > li > a').each do |node|11 css('h2:contains("Nginx API for Lua") + ul > li > a').each do |node|
12 next if node.content == 'Introduction'12 next if node.content == 'Introduction'
13 entries << [node.content, node['href'].remove('#'), 'Nginx API for Lua']13 entries << [node.content, node['href'].remove('#'), 'Nginx API for Lua']
14 end14 end
lib/docs/filters/sanctuary_def/entries.rb+3−1
@@ -39,8 +39,10 @@ module Docs · 38 unmodified lines
39 when "h3"39 when "h3"
40 type = node.text40 type = node.text
41 when "h4"41 when "h4"
42 # Parent <div>'s ID set in github/clean_html.
43 id = node.parent.attributes["id"].value
42 name = node.text.split(' :: ')[0]44 name = node.text.split(' :: ')[0]
43 id = node.attributes["id"].value45
44 entries << [name, id, type]46 entries << [name, id, type]
45 end47 end
46 end48 end
lib/docs/filters/sanctuary_type_classes/clean_html.rb+0−5
@@ -8,11 +8,6 @@ module Docs · 7 unmodified lines
8 node.name = 'h3'8 node.name = 'h3'
9 }9 }
10 10
11 # correct and unify link ids
12 css('h3').each { |node|
13 node.attributes["id"].value = node.text.split(' :: ')[0]
14 }
15
16 doc11 doc
17 end12 end
18 end13 end
lib/docs/filters/sanctuary_type_classes/entries.rb+2−2
@@ -38,9 +38,9 @@ module Docs · 37 unmodified lines
38 case node.name38 case node.name
39 when "h2"39 when "h2"
40 type = node.text40 type = node.text
41 if node.attributes["id"].value == "type-class-hierarchy"41 if node.parent.attributes["id"]&.value == "type-class-hierarchy"
42 name = node.text42 name = node.text
43 id = node.attributes["id"].value43 id = node.parent.attributes["id"].value
44 entries << [name, id, type]44 entries << [name, id, type]
45 end45 end
46 when "h4"46 when "h4"
lib/docs/scrapers/github.rb+7−1
@@ -16,7 +16,13 @@ module Docs · 15 unmodified lines
16 end16 end
17 17
18 def parse(response)18 def parse(response)
19 parsed = JSON.parse(response.response_body)19 embedded_json = response
20 .response_body
21 .match(/react-app\.embeddedData">(.+?)<\/script>/)
22 &.captures
23 &.first
24 parsed = JSON.parse(embedded_json)
25
20 [parsed['payload']['blob']['richText'], parsed['title']]26 [parsed['payload']['blob']['richText'], parsed['title']]
21 end27 end
22 end28 end
lib/docs/scrapers/koa.rb+2−2
@@ -2,7 +2,7 @@ · 1 unmodified line
2 2
3module Docs3module Docs
4 class Koa < Github4 class Koa < Github
5 self.base_url = 'https://github.com/koajs/koa/tree/master/docs'5 self.base_url = 'https://github.com/koajs/koa/blob/master/docs'
6 self.release = '2.15.0'6 self.release = '2.15.0'
7 7
8 self.root_path = 'api/index.md'8 self.root_path = 'api/index.md'
@@ -26,7 +26,7 @@ module Docs · 17 unmodified lines
26 26
27 html_filters.push 'koa/clean_html', 'koa/entries'27 html_filters.push 'koa/clean_html', 'koa/entries'
28 28
29 options[:skip] = %w[middleware.gif]29 options[:skip_patterns] = [/\.gif/]
30 options[:trailing_slash] = false30 options[:trailing_slash] = false
31 options[:container] = '.markdown-body'31 options[:container] = '.markdown-body'
32 32
lib/docs/scrapers/nginx_lua_module.rb+7−5
@@ -2,8 +2,9 @@ module Docs · 1 unmodified line
2 class NginxLuaModule < Github2 class NginxLuaModule < Github
3 self.name = 'nginx / Lua Module'3 self.name = 'nginx / Lua Module'
4 self.slug = 'nginx_lua_module'4 self.slug = 'nginx_lua_module'
5 self.release = '0.10.13'5 self.release = '0.10.28'
6 self.base_url = "https://github.com/openresty/lua-nginx-module/tree/v#{self.release}/"6 self.base_url = "https://github.com/openresty/lua-nginx-module/blob/v#{self.release}/"
7 self.root_path = 'README.markdown'
7 self.links = {8 self.links = {
8 code: 'https://github.com/openresty/lua-nginx-module'9 code: 'https://github.com/openresty/lua-nginx-module'
9 }10 }
@@ -11,13 +12,14 @@ module Docs · 1 unmodified line
11 html_filters.push 'nginx_lua_module/clean_html', 'nginx_lua_module/entries', 'title'12 html_filters.push 'nginx_lua_module/clean_html', 'nginx_lua_module/entries', 'title'
12 13
13 options[:root_title] = 'ngx_http_lua_module'14 options[:root_title] = 'ngx_http_lua_module'
14 options[:container] = '#readme > article'15 options[:container] = '.markdown-body'
15 16 options[:max_image_size] = 256_000
16 options[:attribution] = <<-HTML17 options[:attribution] = <<-HTML
17 &copy; 2009&ndash;2017 Xiaozhe Wang (chaoslawful)<br>18 &copy; 2009&ndash;2017 Xiaozhe Wang (chaoslawful)<br>
18 &copy; 2009&ndash;2018 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.<br>19 &copy; 2009&ndash;2019 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.<br>
19 Licensed under the BSD License.20 Licensed under the BSD License.
20 HTML21 HTML
22 options[:skip_patterns] = [/\.png/]
21 23
22 def get_latest_version(opts)24 def get_latest_version(opts)
23 tags = get_github_tags('openresty', 'lua-nginx-module', opts)25 tags = get_github_tags('openresty', 'lua-nginx-module', opts)
lib/docs/scrapers/q.rb+2−2
@@ -1,5 +1,5 @@
1module Docs1module Docs
2 class Q < Github2 class Q < UrlScraper
3 self.name = 'Q'3 self.name = 'Q'
4 self.release = '1.5.1'4 self.release = '1.5.1'
5 self.base_url = 'https://github.com/kriskowal/q/wiki/'5 self.base_url = 'https://github.com/kriskowal/q/wiki/'
@@ -16,7 +16,7 @@ module Docs · 10 unmodified lines
16 options[:skip_links] = true16 options[:skip_links] = true
17 17
18 options[:attribution] = <<-HTML18 options[:attribution] = <<-HTML
19 &copy; 2009&ndash;2017 Kristopher Michael Kowal<br>19 &copy; 2009&ndash;2018 Kristopher Michael Kowal<br>
20 Licensed under the MIT License.20 Licensed under the MIT License.
21 HTML21 HTML
22 22