Cleanup and improve TypeScript scraper
Unified · Split
| @@ -2,20 +2,23 @@ module Docs · 1 unmodified line | ||
| 2 | 2 | class Typescript |
| 3 | 3 | class CleanHtmlFilter < Filter |
| 4 | 4 | def call |
| 5 | + root_page? ? root : other | |
| 6 | + doc | |
| 7 | + end | |
| 5 | 8 | |
| 6 | − # Top menu bar | |
| 7 | − css('#top-menu').remove | |
| 8 | − css('.skip-to-main').remove | |
| 9 | + def root | |
| 10 | + header = at_css('h1') | |
| 11 | + header.parent.before(header).remove | |
| 9 | 12 | |
| 10 | − # Sidebar | |
| 11 | − css('#sidebar').remove | |
| 13 | + css('h4').each do |node| | |
| 14 | + node.name = 'h2' | |
| 15 | + end | |
| 16 | + end | |
| 12 | 17 | |
| 13 | − # Pound symbol before each title | |
| 14 | − css('.anchor').remove | |
| 18 | + def other | |
| 19 | + @doc = at_css('article > .whitespace > .markdown') | |
| 15 | 20 | |
| 16 | − css('#handbook-content > h2').each do |node| | |
| 17 | − node.name = 'h1' | |
| 18 | − end | |
| 21 | + css('.anchor').remove | |
| 19 | 22 | |
| 20 | 23 | css('a:contains("Try")').remove |
| 21 | 24 | css('pre').each do |node| |
| @@ -23,17 +26,6 @@ module Docs · 1 unmodified line | ||
| 23 | 26 | node['data-language'] = 'typescript' |
| 24 | 27 | node.remove_attribute('class') |
| 25 | 28 | end |
| 26 | − | |
| 27 | − # 'Next' title area | |
| 28 | − css('.whitespace-tight').remove | |
| 29 | − | |
| 30 | − # Right side floating box | |
| 31 | − css('.handbook-toc').remove | |
| 32 | − | |
| 33 | − css('#site-footer').remove | |
| 34 | − | |
| 35 | − doc | |
| 36 | − | |
| 37 | 29 | end |
| 38 | 30 | end |
| 39 | 31 | end |
| @@ -3,7 +3,6 @@ module Docs · 2 unmodified lines | ||
| 3 | 3 | class EntriesFilter < Docs::EntriesFilter |
| 4 | 4 | |
| 5 | 5 | def get_name |
| 6 | − return 'Typescript' if current_url == root_url | |
| 7 | 6 | return at_css('h2').content |
| 8 | 7 | end |
| 9 | 8 | |
| @@ -12,13 +11,9 @@ module Docs · 2 unmodified lines | ||
| 12 | 11 | end |
| 13 | 12 | |
| 14 | 13 | def additional_entries |
| 15 | − entries = [] | |
| 16 | − | |
| 17 | − css('h2').each do |node| | |
| 18 | − entries << [node.content, node['id'], name] | |
| 14 | + css('h2').each_with_object [] do |node,entries| | |
| 15 | + entries << [node.content, node['id'], name] | |
| 19 | 16 | end |
| 20 | − | |
| 21 | − entries | |
| 22 | 17 | end |
| 23 | 18 | |
| 24 | 19 | end |
| @@ -3,23 +3,25 @@ module Docs · 2 unmodified lines | ||
| 3 | 3 | self.name = 'TypeScript' |
| 4 | 4 | self.type = 'simple' |
| 5 | 5 | self.release = '4.1.2' |
| 6 | − self.base_url = 'https://www.typescriptlang.org/docs/handbook' | |
| 6 | + self.base_url = 'https://www.typescriptlang.org/docs/handbook/' | |
| 7 | 7 | self.root_path = 'index.html' |
| 8 | 8 | self.links = { |
| 9 | 9 | home: 'https://www.typescriptlang.org', |
| 10 | 10 | code: 'https://github.com/Microsoft/TypeScript' |
| 11 | 11 | } |
| 12 | 12 | |
| 13 | − html_filters.push 'typescript/entries', 'typescript/clean_html' | |
| 13 | + html_filters.push 'typescript/entries', 'typescript/clean_html', 'title' | |
| 14 | + | |
| 15 | + options[:container] = 'main' | |
| 14 | 16 | |
| 15 | 17 | options[:skip] = [ |
| 16 | − '/react-&-webpack.html', | |
| 17 | − '/asp-net-core.html', | |
| 18 | − '/gulp.html', | |
| 19 | − '/dom-manipulation.html', | |
| 20 | − '/migrating-from-javascript.html', | |
| 21 | − '/babel-with-typescript.html', | |
| 22 | − '/intro.html' | |
| 18 | + 'react-&-webpack.html', | |
| 19 | + 'asp-net-core.html', | |
| 20 | + 'gulp.html', | |
| 21 | + 'dom-manipulation.html', | |
| 22 | + 'migrating-from-javascript.html', | |
| 23 | + 'babel-with-typescript.html', | |
| 24 | + 'intro.html' | |
| 23 | 25 | ] |
| 24 | 26 | |
| 25 | 27 | options[:skip_patterns] = [ |