Switch to UrlScraper Refactoring
Showing the changes to lib/docs (3 of 8 files). Show all
Unified · Split
| @@ -0,0 +1,71 @@ | |||
| 1 | require 'net/http' | ||
| 2 | |||
| 3 | module Docs | ||
| 4 | class Mariadb | ||
| 5 | class CleanHtmlFilter < Filter | ||
| 6 | @@known_urls = Hash.new | ||
| 7 | |||
| 8 | def call | ||
| 9 | # Extract main content | ||
| 10 | @doc = at_css('#content') | ||
| 11 | |||
| 12 | # Remove navigation at the bottom | ||
| 13 | css('.simple_section_nav').remove | ||
| 14 | |||
| 15 | # Remove table of contents | ||
| 16 | css('.table_of_contents').remove | ||
| 17 | |||
| 18 | # Add code highlighting and remove nested tags | ||
| 19 | css('pre').each do |node| | ||
| 20 | node.content = node.content | ||
| 21 | node['data-language'] = 'sql' | ||
| 22 | end | ||
| 23 | |||
| 24 | # Fix links like http://kb-mirror.mariadb.com/kb/en/bitwise-or/ to not redirect to an external page | ||
| 25 | css('a').each do |node| | ||
| 26 | url = node['href'] | ||
| 27 | |||
| 28 | if /^http:\/\/kb-mirror\.mariadb\.com\/kb\/en\/[^\/]+\/(#[^\/]+)?$/.match?(url) | ||
| 29 | final_url = get_final_url(url) | ||
| 30 | |||
| 31 | if !final_url.nil? && final_url.start_with?('/kb/en/library/documentation/') | ||
| 32 | node['href'] = "#{'../' * subpath.count('/')}#{final_url[29..-1]}index" | ||
| 33 | end | ||
| 34 | end | ||
| 35 | end | ||
| 36 | |||
| 37 | # Remove navigation items containing only numbers | ||
| 38 | css('.node_comments').each do |node| | ||
| 39 | if node.content.scan(/\D/).empty? | ||
| 40 | node.remove | ||
| 41 | end | ||
| 42 | end | ||
| 43 | |||
| 44 | # Convert listings (pages like http://kb-mirror.mariadb.com/kb/en/library/documentation/sql-statements-structure/) into tables | ||
| 45 | css('ul.listing').each do |node| | ||
| 46 | rows = [] | ||
| 47 | |||
| 48 | node.css('li').each do |li| | ||
| 49 | name = li.at_css('.media-heading').content | ||
| 50 | description = li.at_css('.blurb').content | ||
| 51 | url = li.at_css('a')['href'] | ||
| 52 | rows << "<tr><td><a href=\"#{url}\">#{name}</a></td><td>#{description}</td></tr>" | ||
| 53 | end | ||
| 54 | |||
| 55 | table = "<table><thead><tr><th>Title</th><th>Description</th></tr></thead><tbody>#{rows.join('')}</tbody></table>" | ||
| 56 | node.replace(table) | ||
| 57 | end | ||
| 58 | |||
| 59 | doc | ||
| 60 | end | ||
| 61 | |||
| 62 | def get_final_url(url) | ||
| 63 | unless @@known_urls.has_key?(url) | ||
| 64 | @@known_urls[url] = Net::HTTP.get_response(URI(url))['location'] | ||
| 65 | end | ||
| 66 | |||
| 67 | @@known_urls[url] | ||
| 68 | end | ||
| 69 | end | ||
| 70 | end | ||
| 71 | end | ||
| @@ -0,0 +1,14 @@ | |||
| 1 | module Docs | ||
| 2 | class Mariadb | ||
| 3 | class EntriesFilter < Docs::EntriesFilter | ||
| 4 | def get_name | ||
| 5 | at_css('.container > h1').content.strip | ||
| 6 | end | ||
| 7 | |||
| 8 | def get_type | ||
| 9 | link = at_css('#breadcrumbs > a:nth-child(6)') | ||
| 10 | link.nil? ? at_css('#breadcrumbs > a:nth-child(5)').content : link.content | ||
| 11 | end | ||
| 12 | end | ||
| 13 | end | ||
| 14 | end | ||
| @@ -0,0 +1,22 @@ | |||
| 1 | module Docs | ||
| 2 | class Mariadb < UrlScraper | ||
| 3 | self.name = 'MariaDB' | ||
| 4 | self.type = 'mariadb' | ||
| 5 | self.release = '10.3.8' | ||
| 6 | self.base_url = 'http://kb-mirror.mariadb.com/kb/en/library/documentation/' | ||
| 7 | self.links = { | ||
| 8 | home: 'https://mariadb.com/', | ||
| 9 | code: 'https://github.com/MariaDB/server' | ||
| 10 | } | ||
| 11 | |||
| 12 | html_filters.push 'mariadb/entries', 'mariadb/clean_html', 'title' | ||
| 13 | |||
| 14 | options[:download_images] = false | ||
| 15 | options[:root_title] = 'MariaDB' | ||
| 16 | |||
| 17 | options[:attribution] = <<-HTML | ||
| 18 | © 2018 MariaDB<br> | ||
| 19 | Licensed under the Creative Commons Attribution 3.0 Unported License and the GNU Free Documentation License. | ||
| 20 | HTML | ||
| 21 | end | ||
| 22 | end | ||