History

Merge pull request #1406 from scherepn/typescript-fix

Cleanup and improve TypeScript scraper

Simon Legner committed 5 years ago

Commit 33125d22 parents 58ebf44 7e368483 files · +26 −37Browse files

Unified · Split

3 files
  1. lib/docs/filters/typescript/clean_html.rb +13 −21
  2. lib/docs/filters/typescript/entries.rb +2 −7
  3. lib/docs/scrapers/typescript.rb +11 −9
lib/docs/filters/typescript/clean_html.rb+13−21
@@ -2,20 +2,23 @@ module Docs · 1 unmodified line
2 class Typescript2 class Typescript
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 root_page? ? root : other
6 doc
7 end
5 8
6 # Top menu bar9 def root
7 css('#top-menu').remove10 header = at_css('h1')
8 css('.skip-to-main').remove11 header.parent.before(header).remove
9 12
10 # Sidebar13 css('h4').each do |node|
11 css('#sidebar').remove14 node.name = 'h2'
15 end
16 end
12 17
13 # Pound symbol before each title18 def other
14 css('.anchor').remove19 @doc = at_css('article > .whitespace > .markdown')
15 20
16 css('#handbook-content > h2').each do |node|21 css('.anchor').remove
17 node.name = 'h1'
18 end
19 22
20 css('a:contains("Try")').remove23 css('a:contains("Try")').remove
21 css('pre').each do |node|24 css('pre').each do |node|
@@ -23,17 +26,6 @@ module Docs · 1 unmodified line
23 node['data-language'] = 'typescript'26 node['data-language'] = 'typescript'
24 node.remove_attribute('class')27 node.remove_attribute('class')
25 end28 end
26
27 # 'Next' title area
28 css('.whitespace-tight').remove
29
30 # Right side floating box
31 css('.handbook-toc').remove
32
33 css('#site-footer').remove
34
35 doc
36
37 end29 end
38 end30 end
39 end31 end
lib/docs/filters/typescript/entries.rb+2−7
@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 4
5 def get_name5 def get_name
6 return 'Typescript' if current_url == root_url
7 return at_css('h2').content6 return at_css('h2').content
8 end7 end
9 8
@@ -12,13 +11,9 @@ module Docs · 2 unmodified lines
12 end11 end
13 12
14 def additional_entries13 def additional_entries
15 entries = []14 css('h2').each_with_object [] do |node,entries|
16 15 entries << [node.content, node['id'], name]
17 css('h2').each do |node|
18 entries << [node.content, node['id'], name]
19 end16 end
20
21 entries
22 end17 end
23 18
24 end19 end
lib/docs/scrapers/typescript.rb+11−9
@@ -3,23 +3,25 @@ module Docs · 2 unmodified lines
3 self.name = 'TypeScript'3 self.name = 'TypeScript'
4 self.type = 'simple'4 self.type = 'simple'
5 self.release = '4.1.2'5 self.release = '4.1.2'
6 self.base_url = 'https://www.typescriptlang.org/docs/handbook'6 self.base_url = 'https://www.typescriptlang.org/docs/handbook/'
7 self.root_path = 'index.html'7 self.root_path = 'index.html'
8 self.links = {8 self.links = {
9 home: 'https://www.typescriptlang.org',9 home: 'https://www.typescriptlang.org',
10 code: 'https://github.com/Microsoft/TypeScript'10 code: 'https://github.com/Microsoft/TypeScript'
11 }11 }
12 12
13 html_filters.push 'typescript/entries', 'typescript/clean_html'13 html_filters.push 'typescript/entries', 'typescript/clean_html', 'title'
14
15 options[:container] = 'main'
14 16
15 options[:skip] = [17 options[:skip] = [
16 '/react-&-webpack.html',18 'react-&-webpack.html',
17 '/asp-net-core.html',19 'asp-net-core.html',
18 '/gulp.html',20 'gulp.html',
19 '/dom-manipulation.html',21 'dom-manipulation.html',
20 '/migrating-from-javascript.html',22 'migrating-from-javascript.html',
21 '/babel-with-typescript.html',23 'babel-with-typescript.html',
22 '/intro.html'24 'intro.html'
23 ]25 ]
24 26
25 options[:skip_patterns] = [27 options[:skip_patterns] = [