Switch to UrlScraper Refactoring
Showing the changes to lib/docs (3 of 8 files). Show all
Unified · Split
| @@ -0,0 +1,71 @@ | ||
| 1 | +require 'net/http' | |
| 2 | + | |
| 3 | +module Docs | |
| 4 | + class Mariadb | |
| 5 | + class CleanHtmlFilter < Filter | |
| 6 | + @@known_urls = Hash.new | |
| 7 | + | |
| 8 | + def call | |
| 9 | + # Extract main content | |
| 10 | + @doc = at_css('#content') | |
| 11 | + | |
| 12 | + # Remove navigation at the bottom | |
| 13 | + css('.simple_section_nav').remove | |
| 14 | + | |
| 15 | + # Remove table of contents | |
| 16 | + css('.table_of_contents').remove | |
| 17 | + | |
| 18 | + # Add code highlighting and remove nested tags | |
| 19 | + css('pre').each do |node| | |
| 20 | + node.content = node.content | |
| 21 | + node['data-language'] = 'sql' | |
| 22 | + end | |
| 23 | + | |
| 24 | + # Fix links like http://kb-mirror.mariadb.com/kb/en/bitwise-or/ to not redirect to an external page | |
| 25 | + css('a').each do |node| | |
| 26 | + url = node['href'] | |
| 27 | + | |
| 28 | + if /^http:\/\/kb-mirror\.mariadb\.com\/kb\/en\/[^\/]+\/(#[^\/]+)?$/.match?(url) | |
| 29 | + final_url = get_final_url(url) | |
| 30 | + | |
| 31 | + if !final_url.nil? && final_url.start_with?('/kb/en/library/documentation/') | |
| 32 | + node['href'] = "#{'../' * subpath.count('/')}#{final_url[29..-1]}index" | |
| 33 | + end | |
| 34 | + end | |
| 35 | + end | |
| 36 | + | |
| 37 | + # Remove navigation items containing only numbers | |
| 38 | + css('.node_comments').each do |node| | |
| 39 | + if node.content.scan(/\D/).empty? | |
| 40 | + node.remove | |
| 41 | + end | |
| 42 | + end | |
| 43 | + | |
| 44 | + # Convert listings (pages like http://kb-mirror.mariadb.com/kb/en/library/documentation/sql-statements-structure/) into tables | |
| 45 | + css('ul.listing').each do |node| | |
| 46 | + rows = [] | |
| 47 | + | |
| 48 | + node.css('li').each do |li| | |
| 49 | + name = li.at_css('.media-heading').content | |
| 50 | + description = li.at_css('.blurb').content | |
| 51 | + url = li.at_css('a')['href'] | |
| 52 | + rows << "<tr><td><a href=\"#{url}\">#{name}</a></td><td>#{description}</td></tr>" | |
| 53 | + end | |
| 54 | + | |
| 55 | + table = "<table><thead><tr><th>Title</th><th>Description</th></tr></thead><tbody>#{rows.join('')}</tbody></table>" | |
| 56 | + node.replace(table) | |
| 57 | + end | |
| 58 | + | |
| 59 | + doc | |
| 60 | + end | |
| 61 | + | |
| 62 | + def get_final_url(url) | |
| 63 | + unless @@known_urls.has_key?(url) | |
| 64 | + @@known_urls[url] = Net::HTTP.get_response(URI(url))['location'] | |
| 65 | + end | |
| 66 | + | |
| 67 | + @@known_urls[url] | |
| 68 | + end | |
| 69 | + end | |
| 70 | + end | |
| 71 | +end | |
| @@ -0,0 +1,14 @@ | ||
| 1 | +module Docs | |
| 2 | + class Mariadb | |
| 3 | + class EntriesFilter < Docs::EntriesFilter | |
| 4 | + def get_name | |
| 5 | + at_css('.container > h1').content.strip | |
| 6 | + end | |
| 7 | + | |
| 8 | + def get_type | |
| 9 | + link = at_css('#breadcrumbs > a:nth-child(6)') | |
| 10 | + link.nil? ? at_css('#breadcrumbs > a:nth-child(5)').content : link.content | |
| 11 | + end | |
| 12 | + end | |
| 13 | + end | |
| 14 | +end | |
| @@ -0,0 +1,22 @@ | ||
| 1 | +module Docs | |
| 2 | + class Mariadb < UrlScraper | |
| 3 | + self.name = 'MariaDB' | |
| 4 | + self.type = 'mariadb' | |
| 5 | + self.release = '10.3.8' | |
| 6 | + self.base_url = 'http://kb-mirror.mariadb.com/kb/en/library/documentation/' | |
| 7 | + self.links = { | |
| 8 | + home: 'https://mariadb.com/', | |
| 9 | + code: 'https://github.com/MariaDB/server' | |
| 10 | + } | |
| 11 | + | |
| 12 | + html_filters.push 'mariadb/entries', 'mariadb/clean_html', 'title' | |
| 13 | + | |
| 14 | + options[:download_images] = false | |
| 15 | + options[:root_title] = 'MariaDB' | |
| 16 | + | |
| 17 | + options[:attribution] = <<-HTML | |
| 18 | + © 2018 MariaDB<br> | |
| 19 | + Licensed under the Creative Commons Attribution 3.0 Unported License and the GNU Free Documentation License. | |
| 20 | + HTML | |
| 21 | + end | |
| 22 | +end | |