History

Merge pull request #841 from jmerle/mariadb

Add MariaDB documentation

Jasper van Merle committed 7 years ago

Commit c8380bb2 parents fdb9701 5ed994210 files · +173 −-2Browse files

Unified · Split

10 files
  1. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  2. assets/stylesheets/application.css.scss +1 −0
  3. assets/stylesheets/pages/_mariadb.scss +7 −0
  4. lib/docs/filters/mariadb/clean_html.rb +67 −0
  5. lib/docs/filters/mariadb/entries.rb +24 −0
  6. lib/docs/filters/mariadb/erase_invalid_pages.rb +34 −0
  7. lib/docs/scrapers/mariadb.rb +36 −0
  8. public/icons/docs/mariadb/16.png +0 −0
  9. public/icons/docs/mariadb/16@2x.png +0 −0
  10. public/icons/docs/mariadb/SOURCE +1 −0
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -451,6 +451,11 @@ credits = [ · 450 unmodified lines
451451 '2006-2016 LÖVE Development Team',
452452 'GFDL',
453453 'http://www.gnu.org/copyleft/fdl.html'
454+ ], [
455+ 'MariaDB',
456+ '2019 MariaDB',
457+ 'CC BY-SA & GFDL',
458+ 'https://mariadb.com/kb/en/library/documentation/+license/'
454459 ], [
455460 'Marionette.js',
456461 '2017 Muted Solutions, LLC',
assets/stylesheets/application.css.scss+1−0
@@ -70,6 +70,7 @@ · 69 unmodified lines
7070 'pages/liquid',
7171 'pages/love',
7272 'pages/lua',
73+ 'pages/mariadb',
7374 'pages/mdn',
7475 'pages/meteor',
7576 'pages/mkdocs',
assets/stylesheets/pages/_mariadb.scss+7−0
@@ -0,0 +1,7 @@
1+._mariadb {
2+ @extend %simple;
3+
4+ .graybox, .product {
5+ @extend %note;
6+ }
7+}
lib/docs/filters/mariadb/clean_html.rb+67−0
@@ -0,0 +1,67 @@
1+module Docs
2+ class Mariadb
3+ class CleanHtmlFilter < Filter
4+ def call
5+ # Return the empty doc if the EraseInvalidPagesFilter detected this page shouldn't be scraped
6+ return doc if doc.inner_html == ''
7+
8+ # Extract main content
9+ @doc = at_css('#content')
10+
11+ # Remove navigation at the bottom
12+ css('.simple_section_nav').remove
13+
14+ # Remove table of contents
15+ css('.table_of_contents').remove
16+
17+ # Add code highlighting and remove nested tags
18+ css('pre').each do |node|
19+ node.content = node.content
20+ node['data-language'] = 'sql'
21+ end
22+
23+ # Fix images
24+ css('img').each do |node|
25+ node['src'] = node['src'].sub('http:', 'https:')
26+ end
27+
28+ # Remove navigation items containing only numbers
29+ css('.node_comments').each do |node|
30+ if node.content.scan(/\D/).empty?
31+ node.remove
32+ end
33+ end
34+
35+ # Convert listings (pages like https://mariadb.com/kb/en/library/documentation/sql-statements-structure/) into tables
36+ css('ul.listing').each do |node|
37+ rows = []
38+
39+ node.css('li:not(.no_data)').each do |li|
40+ name = li.at_css('.media-heading').content
41+ description = li.at_css('.blurb').content
42+ url = li.at_css('a')['href']
43+ rows << "<tr><td><a href=\"#{url}\">#{name}</a></td><td>#{description}</td></tr>"
44+ end
45+
46+ table = "<table><thead><tr><th>Title</th><th>Description</th></tr></thead><tbody>#{rows.join('')}</tbody></table>"
47+ node.replace(table)
48+ end
49+
50+ # Turn note titles into <strong> tags
51+ css('.product_title').each do |node|
52+ node.name = 'strong'
53+ end
54+
55+ # Remove comments and questions
56+ css('.related_questions, #comments').remove
57+ css('h2').each do |node|
58+ if node.content == 'Comments'
59+ node.remove
60+ end
61+ end
62+
63+ doc
64+ end
65+ end
66+ end
67+end
lib/docs/filters/mariadb/entries.rb+24−0
@@ -0,0 +1,24 @@
1+module Docs
2+ class Mariadb
3+ class EntriesFilter < Docs::EntriesFilter
4+ def get_name
5+ return 'Name' if doc.inner_html == ''
6+
7+ at_css('#content > h1').content.strip
8+ end
9+
10+ def get_type
11+ return 'Type' if doc.inner_html == ''
12+
13+ link = at_css('#breadcrumbs > a:nth-child(4)')
14+ link.nil? ? at_css('#breadcrumbs > a:nth-child(3)').content : link.content
15+ end
16+
17+ def entries
18+ # Don't add an entry for this page if the EraseInvalidPagesFilter detected this page shouldn't be scraped
19+ return [] if doc.inner_html == ''
20+ super
21+ end
22+ end
23+ end
24+end
lib/docs/filters/mariadb/erase_invalid_pages.rb+34−0
@@ -0,0 +1,34 @@
1+module Docs
2+ class Mariadb
3+ class EraseInvalidPagesFilter < Filter
4+ @@seen_urls = Hash.new
5+
6+ def call
7+ # The MariaDB documentation uses urls like mariadb.com/kb/en/*
8+ # This means there is no way to detect if a page should be scraped based on it's url
9+ # We run this filter before the internal_urls filter scrapes all internal urls
10+ # If this page should not be scraped, we erase it's contents in here so that the internal urls are not picked up
11+ # The entries filter will make sure that no entry is saved for this page
12+
13+ if at_css('a.crumb[href="https://mariadb.com/kb/en/documentation/"]').nil?
14+ doc.inner_html = ''
15+ end
16+
17+ current_page = at_css('a.crumb.node_link')
18+ unless current_page.nil?
19+ url = current_page['href']
20+
21+ # Some links lead to the same page
22+ # Only parse the page one time
23+ if @@seen_urls.has_key?(url)
24+ doc.inner_html = ''
25+ end
26+
27+ @@seen_urls[url] = true
28+ end
29+
30+ doc
31+ end
32+ end
33+ end
34+end
lib/docs/scrapers/mariadb.rb+36−0
@@ -0,0 +1,36 @@
1+module Docs
2+ class Mariadb < UrlScraper
3+ self.name = 'MariaDB'
4+ self.type = 'mariadb'
5+ self.release = '10.4.8'
6+ self.base_url = 'https://mariadb.com/kb/en/'
7+ self.root_path = 'library/documentation/'
8+ self.links = {
9+ home: 'https://mariadb.com/',
10+ code: 'https://github.com/MariaDB/server'
11+ }
12+
13+ html_filters.insert_before 'internal_urls', 'mariadb/erase_invalid_pages'
14+ html_filters.push 'mariadb/entries', 'mariadb/clean_html'
15+
16+ options[:rate_limit] = 200
17+ options[:skip_patterns] = [
18+ /\+/,
19+ /\/ask\//,
20+ /-release-notes\//,
21+ /-changelog\//,
22+ /^documentation\//,
23+ /^mariadb-server-documentation\//,
24+ ]
25+
26+ options[:attribution] = <<-HTML
27+ &copy; 2019 MariaDB<br>
28+ Licensed under the Creative Commons Attribution 3.0 Unported License and the GNU Free Documentation License.
29+ HTML
30+
31+ def get_latest_version(opts)
32+ doc = fetch_doc('https://mariadb.com/downloads/', opts)
33+ doc.at_css('[data-version-id="mariadb_server-versions"] option').content.split('-')[0]
34+ end
35+ end
36+end
public/icons/docs/mariadb/16.png+0−0

Binary file not shown.

public/icons/docs/mariadb/16@2x.png+0−0

Binary file not shown.

public/icons/docs/mariadb/SOURCE+1−0
@@ -0,0 +1 @@
1+https://mariadb.org/about/logos/