History

Add MariaDB documentation

Switch to UrlScraper Refactoring

Jasper van Merle committed 8 years ago

Commit e894a2f1 parent 6ecd9d88 files · +122 −0Browse files

Unified · Split

8 files
  1. .gitignore +1 −0
  2. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  3. assets/stylesheets/application-dark.css.scss +1 −0
  4. assets/stylesheets/application.css.scss +1 −0
  5. assets/stylesheets/pages/_mariadb.scss +7 −0
  6. lib/docs/filters/mariadb/clean_html.rb +71 −0
  7. lib/docs/filters/mariadb/entries.rb +14 −0
  8. lib/docs/scrapers/mariadb.rb +22 −0
.gitignore+1−0
@@ -8,3 +8,4 @@ public/fonts · 7 unmodified lines
8public/docs/**/*8public/docs/**/*
9!public/docs/docs.json9!public/docs/docs.json
10!public/docs/**/index.json10!public/docs/**/index.json
11log
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -410,6 +410,11 @@ credits = [ · 409 unmodified lines
410 '2006-2016 LÖVE Development Team',410 '2006-2016 LÖVE Development Team',
411 'GFDL',411 'GFDL',
412 'http://www.gnu.org/copyleft/fdl.html'412 'http://www.gnu.org/copyleft/fdl.html'
413 ], [
414 'MariaDB',
415 '2018 MariaDB',
416 'CC BY-SA & GFDL',
417 'https://mariadb.com/kb/en/library/documentation/+license/'
413 ], [418 ], [
414 'Marionette.js',419 'Marionette.js',
415 '2017 Muted Solutions, LLC',420 '2017 Muted Solutions, LLC',
assets/stylesheets/application-dark.css.scss+1−0
@@ -64,6 +64,7 @@ · 63 unmodified lines
64 'pages/liquid',64 'pages/liquid',
65 'pages/love',65 'pages/love',
66 'pages/lua',66 'pages/lua',
67 'pages/mariadb',
67 'pages/mdn',68 'pages/mdn',
68 'pages/meteor',69 'pages/meteor',
69 'pages/modernizr',70 'pages/modernizr',
assets/stylesheets/application.css.scss+1−0
@@ -64,6 +64,7 @@ · 63 unmodified lines
64 'pages/liquid',64 'pages/liquid',
65 'pages/love',65 'pages/love',
66 'pages/lua',66 'pages/lua',
67 'pages/mariadb',
67 'pages/mdn',68 'pages/mdn',
68 'pages/meteor',69 'pages/meteor',
69 'pages/modernizr',70 'pages/modernizr',
assets/stylesheets/pages/_mariadb.scss+7−0
@@ -0,0 +1,7 @@
1._mariadb {
2 @extend %simple;
3
4 .graybox {
5 @extend %note;
6 }
7}
lib/docs/filters/mariadb/clean_html.rb+71−0
@@ -0,0 +1,71 @@
1require 'net/http'
2
3module Docs
4 class Mariadb
5 class CleanHtmlFilter < Filter
6 @@known_urls = Hash.new
7
8 def call
9 # Extract main content
10 @doc = at_css('#content')
11
12 # Remove navigation at the bottom
13 css('.simple_section_nav').remove
14
15 # Remove table of contents
16 css('.table_of_contents').remove
17
18 # Add code highlighting and remove nested tags
19 css('pre').each do |node|
20 node.content = node.content
21 node['data-language'] = 'sql'
22 end
23
24 # Fix links like http://kb-mirror.mariadb.com/kb/en/bitwise-or/ to not redirect to an external page
25 css('a').each do |node|
26 url = node['href']
27
28 if /^http:\/\/kb-mirror\.mariadb\.com\/kb\/en\/[^\/]+\/(#[^\/]+)?$/.match?(url)
29 final_url = get_final_url(url)
30
31 if !final_url.nil? && final_url.start_with?('/kb/en/library/documentation/')
32 node['href'] = "#{'../' * subpath.count('/')}#{final_url[29..-1]}index"
33 end
34 end
35 end
36
37 # Remove navigation items containing only numbers
38 css('.node_comments').each do |node|
39 if node.content.scan(/\D/).empty?
40 node.remove
41 end
42 end
43
44 # Convert listings (pages like http://kb-mirror.mariadb.com/kb/en/library/documentation/sql-statements-structure/) into tables
45 css('ul.listing').each do |node|
46 rows = []
47
48 node.css('li').each do |li|
49 name = li.at_css('.media-heading').content
50 description = li.at_css('.blurb').content
51 url = li.at_css('a')['href']
52 rows << "<tr><td><a href=\"#{url}\">#{name}</a></td><td>#{description}</td></tr>"
53 end
54
55 table = "<table><thead><tr><th>Title</th><th>Description</th></tr></thead><tbody>#{rows.join('')}</tbody></table>"
56 node.replace(table)
57 end
58
59 doc
60 end
61
62 def get_final_url(url)
63 unless @@known_urls.has_key?(url)
64 @@known_urls[url] = Net::HTTP.get_response(URI(url))['location']
65 end
66
67 @@known_urls[url]
68 end
69 end
70 end
71end
lib/docs/filters/mariadb/entries.rb+14−0
@@ -0,0 +1,14 @@
1module Docs
2 class Mariadb
3 class EntriesFilter < Docs::EntriesFilter
4 def get_name
5 at_css('.container > h1').content.strip
6 end
7
8 def get_type
9 link = at_css('#breadcrumbs > a:nth-child(6)')
10 link.nil? ? at_css('#breadcrumbs > a:nth-child(5)').content : link.content
11 end
12 end
13 end
14end
lib/docs/scrapers/mariadb.rb+22−0
@@ -0,0 +1,22 @@
1module Docs
2 class Mariadb < UrlScraper
3 self.name = 'MariaDB'
4 self.type = 'mariadb'
5 self.release = '10.3.8'
6 self.base_url = 'http://kb-mirror.mariadb.com/kb/en/library/documentation/'
7 self.links = {
8 home: 'https://mariadb.com/',
9 code: 'https://github.com/MariaDB/server'
10 }
11
12 html_filters.push 'mariadb/entries', 'mariadb/clean_html', 'title'
13
14 options[:download_images] = false
15 options[:root_title] = 'MariaDB'
16
17 options[:attribution] = <<-HTML
18 &copy; 2018 MariaDB<br>
19 Licensed under the Creative Commons Attribution 3.0 Unported License and the GNU Free Documentation License.
20 HTML
21 end
22end