Add MariaDB documentation
Showing the changes to lib/docs (4 of 10 files). Show all
Unified · Split
| @@ -0,0 +1,67 @@ | |||
| 1 | module Docs | ||
| 2 | class Mariadb | ||
| 3 | class CleanHtmlFilter < Filter | ||
| 4 | def call | ||
| 5 | # Return the empty doc if the EraseInvalidPagesFilter detected this page shouldn't be scraped | ||
| 6 | return doc if doc.inner_html == '' | ||
| 7 | |||
| 8 | # Extract main content | ||
| 9 | @doc = at_css('#content') | ||
| 10 | |||
| 11 | # Remove navigation at the bottom | ||
| 12 | css('.simple_section_nav').remove | ||
| 13 | |||
| 14 | # Remove table of contents | ||
| 15 | css('.table_of_contents').remove | ||
| 16 | |||
| 17 | # Add code highlighting and remove nested tags | ||
| 18 | css('pre').each do |node| | ||
| 19 | node.content = node.content | ||
| 20 | node['data-language'] = 'sql' | ||
| 21 | end | ||
| 22 | |||
| 23 | # Fix images | ||
| 24 | css('img').each do |node| | ||
| 25 | node['src'] = node['src'].sub('http:', 'https:') | ||
| 26 | end | ||
| 27 | |||
| 28 | # Remove navigation items containing only numbers | ||
| 29 | css('.node_comments').each do |node| | ||
| 30 | if node.content.scan(/\D/).empty? | ||
| 31 | node.remove | ||
| 32 | end | ||
| 33 | end | ||
| 34 | |||
| 35 | # Convert listings (pages like https://mariadb.com/kb/en/library/documentation/sql-statements-structure/) into tables | ||
| 36 | css('ul.listing').each do |node| | ||
| 37 | rows = [] | ||
| 38 | |||
| 39 | node.css('li:not(.no_data)').each do |li| | ||
| 40 | name = li.at_css('.media-heading').content | ||
| 41 | description = li.at_css('.blurb').content | ||
| 42 | url = li.at_css('a')['href'] | ||
| 43 | rows << "<tr><td><a href=\"#{url}\">#{name}</a></td><td>#{description}</td></tr>" | ||
| 44 | end | ||
| 45 | |||
| 46 | table = "<table><thead><tr><th>Title</th><th>Description</th></tr></thead><tbody>#{rows.join('')}</tbody></table>" | ||
| 47 | node.replace(table) | ||
| 48 | end | ||
| 49 | |||
| 50 | # Turn note titles into <strong> tags | ||
| 51 | css('.product_title').each do |node| | ||
| 52 | node.name = 'strong' | ||
| 53 | end | ||
| 54 | |||
| 55 | # Remove comments and questions | ||
| 56 | css('.related_questions, #comments').remove | ||
| 57 | css('h2').each do |node| | ||
| 58 | if node.content == 'Comments' | ||
| 59 | node.remove | ||
| 60 | end | ||
| 61 | end | ||
| 62 | |||
| 63 | doc | ||
| 64 | end | ||
| 65 | end | ||
| 66 | end | ||
| 67 | end | ||
| @@ -0,0 +1,24 @@ | |||
| 1 | module Docs | ||
| 2 | class Mariadb | ||
| 3 | class EntriesFilter < Docs::EntriesFilter | ||
| 4 | def get_name | ||
| 5 | return 'Name' if doc.inner_html == '' | ||
| 6 | |||
| 7 | at_css('#content > h1').content.strip | ||
| 8 | end | ||
| 9 | |||
| 10 | def get_type | ||
| 11 | return 'Type' if doc.inner_html == '' | ||
| 12 | |||
| 13 | link = at_css('#breadcrumbs > a:nth-child(4)') | ||
| 14 | link.nil? ? at_css('#breadcrumbs > a:nth-child(3)').content : link.content | ||
| 15 | end | ||
| 16 | |||
| 17 | def entries | ||
| 18 | # Don't add an entry for this page if the EraseInvalidPagesFilter detected this page shouldn't be scraped | ||
| 19 | return [] if doc.inner_html == '' | ||
| 20 | super | ||
| 21 | end | ||
| 22 | end | ||
| 23 | end | ||
| 24 | end | ||
| @@ -0,0 +1,34 @@ | |||
| 1 | module Docs | ||
| 2 | class Mariadb | ||
| 3 | class EraseInvalidPagesFilter < Filter | ||
| 4 | @@seen_urls = Hash.new | ||
| 5 | |||
| 6 | def call | ||
| 7 | # The MariaDB documentation uses urls like mariadb.com/kb/en/* | ||
| 8 | # This means there is no way to detect if a page should be scraped based on it's url | ||
| 9 | # We run this filter before the internal_urls filter scrapes all internal urls | ||
| 10 | # If this page should not be scraped, we erase it's contents in here so that the internal urls are not picked up | ||
| 11 | # The entries filter will make sure that no entry is saved for this page | ||
| 12 | |||
| 13 | if at_css('a.crumb[href="https://mariadb.com/kb/en/documentation/"]').nil? | ||
| 14 | doc.inner_html = '' | ||
| 15 | end | ||
| 16 | |||
| 17 | current_page = at_css('a.crumb.node_link') | ||
| 18 | unless current_page.nil? | ||
| 19 | url = current_page['href'] | ||
| 20 | |||
| 21 | # Some links lead to the same page | ||
| 22 | # Only parse the page one time | ||
| 23 | if @@seen_urls.has_key?(url) | ||
| 24 | doc.inner_html = '' | ||
| 25 | end | ||
| 26 | |||
| 27 | @@seen_urls[url] = true | ||
| 28 | end | ||
| 29 | |||
| 30 | doc | ||
| 31 | end | ||
| 32 | end | ||
| 33 | end | ||
| 34 | end | ||
| @@ -0,0 +1,36 @@ | |||
| 1 | module Docs | ||
| 2 | class Mariadb < UrlScraper | ||
| 3 | self.name = 'MariaDB' | ||
| 4 | self.type = 'mariadb' | ||
| 5 | self.release = '10.4.8' | ||
| 6 | self.base_url = 'https://mariadb.com/kb/en/' | ||
| 7 | self.root_path = 'library/documentation/' | ||
| 8 | self.links = { | ||
| 9 | home: 'https://mariadb.com/', | ||
| 10 | code: 'https://github.com/MariaDB/server' | ||
| 11 | } | ||
| 12 | |||
| 13 | html_filters.insert_before 'internal_urls', 'mariadb/erase_invalid_pages' | ||
| 14 | html_filters.push 'mariadb/entries', 'mariadb/clean_html' | ||
| 15 | |||
| 16 | options[:rate_limit] = 200 | ||
| 17 | options[:skip_patterns] = [ | ||
| 18 | /\+/, | ||
| 19 | /\/ask\//, | ||
| 20 | /-release-notes\//, | ||
| 21 | /-changelog\//, | ||
| 22 | /^documentation\//, | ||
| 23 | /^mariadb-server-documentation\//, | ||
| 24 | ] | ||
| 25 | |||
| 26 | options[:attribution] = <<-HTML | ||
| 27 | © 2019 MariaDB<br> | ||
| 28 | Licensed under the Creative Commons Attribution 3.0 Unported License and the GNU Free Documentation License. | ||
| 29 | HTML | ||
| 30 | |||
| 31 | def get_latest_version(opts) | ||
| 32 | doc = fetch_doc('https://mariadb.com/downloads/', opts) | ||
| 33 | doc.at_css('[data-version-id="mariadb_server-versions"] option').content.split('-')[0] | ||
| 34 | end | ||
| 35 | end | ||
| 36 | end | ||