Showing the changes to lib/docs (5 of 6 files). Show all
Unified · Split
| @@ -3,7 +3,7 @@ require 'set' · 2 unmodified lines | ||
| 3 | 3 | module Docs |
| 4 | 4 | class Scraper < Doc |
| 5 | 5 | class << self |
| 6 | − attr_accessor :base_url, :root_path, :initial_paths, :options, :html_filters, :text_filters, :stubs | |
| 6 | + attr_accessor :base_url, :root_path, :initial_paths, :initial_urls, :options, :html_filters, :text_filters, :stubs | |
| 7 | 7 | |
| 8 | 8 | def inherited(subclass) |
| 9 | 9 | super |
| @@ -16,6 +16,7 @@ module Docs · 6 unmodified lines | ||
| 16 | 16 | subclass.base_url = base_url |
| 17 | 17 | subclass.root_path = root_path |
| 18 | 18 | subclass.initial_paths = initial_paths.dup |
| 19 | + subclass.initial_urls = initial_urls.dup | |
| 19 | 20 | subclass.options = options.deep_dup |
| 20 | 21 | subclass.html_filters = html_filters.inheritable_copy |
| 21 | 22 | subclass.text_filters = text_filters.inheritable_copy |
| @@ -35,6 +36,7 @@ module Docs · 13 unmodified lines | ||
| 35 | 36 | include Instrumentable |
| 36 | 37 | |
| 37 | 38 | self.initial_paths = [] |
| 39 | + self.initial_urls = [] | |
| 38 | 40 | self.options = {} |
| 39 | 41 | self.stubs = {} |
| 40 | 42 | |
| @@ -103,7 +105,7 @@ module Docs · 62 unmodified lines | ||
| 103 | 105 | end |
| 104 | 106 | |
| 105 | 107 | def initial_urls |
| 106 | − @initial_urls ||= [root_url.to_s].concat(initial_paths.map(&method(:url_for))).freeze | |
| 108 | + @initial_urls ||= [root_url.to_s].concat(self.class.initial_urls).concat(initial_paths.map(&method(:url_for))).freeze | |
| 107 | 109 | end |
| 108 | 110 | |
| 109 | 111 | def pipeline |
| @@ -33,7 +33,11 @@ module Docs · 32 unmodified lines | ||
| 33 | 33 | raise "Error status code (#{response.code}): #{response.url}" |
| 34 | 34 | end |
| 35 | 35 | |
| 36 | − response.success? && response.html? && base_url.contains?(response.effective_url) | |
| 36 | + response.success? && response.html? && process_url?(response.effective_url) | |
| 37 | + end | |
| 38 | + | |
| 39 | + def process_url?(url) | |
| 40 | + base_url.contains?(url) | |
| 37 | 41 | end |
| 38 | 42 | |
| 39 | 43 | def load_capybara_selenium |
| @@ -2,44 +2,45 @@ module Docs · 1 unmodified line | ||
| 2 | 2 | class Meteor |
| 3 | 3 | class CleanHtmlFilter < Filter |
| 4 | 4 | def call |
| 5 | − root_page? ? root : other | |
| 5 | + @doc = at_css('.content-wrapper') | |
| 6 | 6 | |
| 7 | − css('pre span').each do |node| | |
| 8 | − node.before(node.children).remove | |
| 9 | − end | |
| 10 | − | |
| 11 | − doc | |
| 12 | − end | |
| 7 | + css('.page-actions', '.anchor').remove | |
| 13 | 8 | |
| 14 | − def root | |
| 15 | − @doc = at_css('#introduction').parent | |
| 16 | − | |
| 17 | − css('.github-ribbon', '#introduction').remove | |
| 18 | − | |
| 19 | − css('.selflink', 'b > em').each do |node| | |
| 9 | + css('.header-content', '.document-formatting', 'h2 > a', '.api', '.api-body', 'div.desc').each do |node| | |
| 20 | 10 | node.before(node.children).remove |
| 21 | 11 | end |
| 22 | 12 | |
| 23 | − css('pre').each do |node| | |
| 24 | − node['data-language'] = node.at_css('code')['class'].include?('html') ? 'html' : 'js' | |
| 25 | − node.content = node.content | |
| 13 | + css('.anchor-offset').each do |node| | |
| 14 | + node.parent['id'] = node['id'] | |
| 15 | + node.remove | |
| 26 | 16 | end |
| 27 | 17 | |
| 28 | − css('a.src-code').each do |node| | |
| 29 | − node.content = 'Source' | |
| 18 | + css('.api-heading').each do |node| | |
| 19 | + heading = node.at_css('h2, h3') | |
| 20 | + name = heading.name | |
| 21 | + node['id'] = heading['id'] | |
| 22 | + heading.replace "<code>#{heading.content.strip}</code>" | |
| 23 | + node.name = name | |
| 30 | 24 | end |
| 31 | − end | |
| 32 | 25 | |
| 33 | − def other | |
| 34 | − @doc = at_css('#content') | |
| 35 | − | |
| 36 | − css('.edit-discuss-links', '.bottom-nav', '.edit-link').remove | |
| 26 | + css('div.code', 'span.code', '.args .name').each do |node| | |
| 27 | + node.name = 'code' | |
| 28 | + node.remove_attribute('class') | |
| 29 | + end | |
| 37 | 30 | |
| 38 | 31 | css('figure.highlight').each do |node| |
| 39 | − node.inner_html = node.at_css('.code pre').inner_html.gsub('<br>', "\n") | |
| 32 | + node.inner_html = node.at_css('.code pre').inner_html.gsub('</div><div', "</div>\n<div").gsub('<br>', "\n") | |
| 33 | + node.content = node.content | |
| 40 | 34 | node['data-language'] = node['class'].split.last |
| 41 | 35 | node.name = 'pre' |
| 42 | 36 | end |
| 37 | + | |
| 38 | + css('pre.prettyprint').each do |node| | |
| 39 | + node['data-language'] = node['class'].include?('html') ? 'html' : 'js' | |
| 40 | + node.content = node.content | |
| 41 | + end | |
| 42 | + | |
| 43 | + doc | |
| 43 | 44 | end |
| 44 | 45 | end |
| 45 | 46 | end |
| @@ -2,32 +2,31 @@ module Docs · 1 unmodified line | ||
| 2 | 2 | class Meteor |
| 3 | 3 | class EntriesFilter < Docs::EntriesFilter |
| 4 | 4 | def get_name |
| 5 | − at_css('#content > h1').content | |
| 5 | + at_css('.item-toc.current').content | |
| 6 | 6 | end |
| 7 | 7 | |
| 8 | 8 | def get_type |
| 9 | − if (node = at_css('#sidebar .current')) && (node = node.ancestors('.menu-root').first.previous_element) | |
| 10 | − "Guide: #{node.content}" | |
| 9 | + if subpath.start_with?('api') | |
| 10 | + name | |
| 11 | 11 | else |
| 12 | − 'Guide' | |
| 12 | + type = at_css('.item-toc.current').ancestors('li').first.at_css('.heading-toc').try(:content) || 'Guide' | |
| 13 | + type.prepend 'Guide: ' if base_url.host == 'guide.meteor.com' && type != 'Guide' | |
| 14 | + type | |
| 13 | 15 | end |
| 14 | 16 | end |
| 15 | 17 | |
| 16 | 18 | def additional_entries |
| 17 | − return [] unless root_page? | |
| 18 | − type = nil | |
| 19 | − | |
| 20 | − at_css('.full-api-toc').element_children.each_with_object [] do |node, entries| | |
| 21 | − link = node.at_css('a') | |
| 22 | − next unless link | |
| 23 | − | |
| 24 | − target = link['href'].remove('#/full/') | |
| 25 | − | |
| 26 | − case node.name | |
| 27 | − when 'h1', 'h2' | |
| 28 | − type = node.content.strip | |
| 29 | − when 'h3', 'h4' | |
| 30 | − entries << [node.content, target, type] | |
| 19 | + if slug == 'commandline' | |
| 20 | + css('h2[id]').map do |node| | |
| 21 | + [node.content, node['id']] | |
| 22 | + end | |
| 23 | + else | |
| 24 | + css('.title-api[id]').map do |node| | |
| 25 | + name = node.content.strip | |
| 26 | + name.sub! %r{\(.+\)}, '()' | |
| 27 | + name.remove! 'new ' | |
| 28 | + name = '{{> Template.dynamic }}' if name.include?('Template.dynamic') | |
| 29 | + [name, node['id']] | |
| 31 | 30 | end |
| 32 | 31 | end |
| 33 | 32 | end |
| @@ -1,9 +1,11 @@ | ||
| 1 | 1 | module Docs |
| 2 | 2 | class Meteor < UrlScraper |
| 3 | + class << self | |
| 4 | + attr_accessor :guide_url | |
| 5 | + end | |
| 6 | + | |
| 3 | 7 | self.type = 'meteor' |
| 4 | − self.release = '1.3.2' | |
| 5 | − self.base_url = 'https://guide.meteor.com/v1.3/' | |
| 6 | − self.initial_paths = %w(guide) | |
| 8 | + self.root_path = 'index.html' | |
| 7 | 9 | self.links = { |
| 8 | 10 | home: 'https://www.meteor.com/', |
| 9 | 11 | code: 'https://github.com/meteor/meteor/' |
| @@ -11,26 +13,52 @@ module Docs · 1 unmodified line | ||
| 11 | 13 | |
| 12 | 14 | html_filters.push 'meteor/entries', 'meteor/clean_html' |
| 13 | 15 | |
| 14 | − options[:skip_links] = ->(filter) { filter.root_page? } | |
| 16 | + options[:skip_patterns] = [/\Av\d/] | |
| 17 | + options[:skip] = %w( | |
| 18 | + CONTRIBUTING.html | |
| 19 | + CHANGELOG.html | |
| 20 | + using-packages.html | |
| 21 | + writing-packages.html | |
| 22 | + ) | |
| 15 | 23 | |
| 16 | 24 | options[:attribution] = <<-HTML |
| 17 | 25 | © 2011–2016 Meteor Development Group<br> |
| 18 | 26 | Licensed under the MIT License. |
| 19 | 27 | HTML |
| 20 | 28 | |
| 21 | − stub '' do | |
| 22 | − require 'capybara/dsl' | |
| 23 | − Capybara.current_driver = :selenium | |
| 24 | − Capybara.run_server = false | |
| 25 | − Capybara.app_host = 'https://docs.meteor.com' | |
| 26 | − Capybara.visit('/#/full/') | |
| 27 | − Capybara.find('.body')['innerHTML'] | |
| 29 | + version '1.4' do | |
| 30 | + self.release = '1.4.0' | |
| 31 | + self.base_url = 'https://docs.meteor.com/' | |
| 32 | + self.guide_url = 'https://guide.meteor.com/' | |
| 33 | + self.initial_urls = [guide_url] | |
| 34 | + end | |
| 35 | + | |
| 36 | + version '1.3' do | |
| 37 | + self.release = '1.3.5' | |
| 38 | + self.base_url = "https://docs.meteor.com/v#{self.release}/" | |
| 39 | + self.guide_url = 'https://guide.meteor.com/v1.3/' | |
| 40 | + self.initial_urls = [guide_url] | |
| 28 | 41 | end |
| 29 | 42 | |
| 30 | − stub 'guide' do | |
| 31 | − request_one(url_for('index.html')).body | |
| 43 | + def guide_url | |
| 44 | + @guide_url ||= URL.parse(self.class.guide_url) | |
| 32 | 45 | end |
| 33 | 46 | |
| 34 | − options[:replace_paths] = { 'index.html' => 'guide' } | |
| 47 | + private | |
| 48 | + | |
| 49 | + def process_url?(url) | |
| 50 | + base_url.contains?(url) || guide_url.contains?(url) | |
| 51 | + end | |
| 52 | + | |
| 53 | + def process_response(response) | |
| 54 | + original_host = @base_url.host | |
| 55 | + original_path = @base_url.path | |
| 56 | + @base_url.host = response.effective_url.host | |
| 57 | + @base_url.path = response.effective_url.path[/\A\/v[\d\.]+\//, 0] || '/' | |
| 58 | + super | |
| 59 | + ensure | |
| 60 | + @base_url.host = original_host | |
| 61 | + @base_url.path = original_path | |
| 62 | + end | |
| 35 | 63 | end |
| 36 | 64 | end |