Showing the changes to lib/docs (8 of 21 files). Show all
Unified · Split
| @@ -1,11 +1,10 @@ | ||
| 1 | 1 | module Docs |
| 2 | 2 | class Parser |
| 3 | + attr_reader :title, :html | |
| 4 | + | |
| 3 | 5 | def initialize(content) |
| 4 | 6 | @content = content |
| 5 | − end | |
| 6 | − | |
| 7 | − def html | |
| 8 | − @html ||= document? ? parse_as_document : parse_as_fragment | |
| 7 | + @html = document? ? parse_as_document : parse_as_fragment | |
| 9 | 8 | end |
| 10 | 9 | |
| 11 | 10 | private |
| @@ -16,6 +15,7 @@ module Docs · 4 unmodified lines | ||
| 16 | 15 | |
| 17 | 16 | def parse_as_document |
| 18 | 17 | document = Nokogiri::HTML.parse @content, nil, 'UTF-8' |
| 18 | + @title = document.at_css('title').try(:content) | |
| 19 | 19 | document.at_css 'body' |
| 20 | 20 | end |
| 21 | 21 | |
| @@ -171,7 +171,10 @@ module Docs · 170 unmodified lines | ||
| 171 | 171 | |
| 172 | 172 | def process_response(response) |
| 173 | 173 | data = {} |
| 174 | − pipeline.call(parse(response.body), pipeline_context(response), data) | |
| 174 | + html, title = parse(response.body) | |
| 175 | + context = pipeline_context(response) | |
| 176 | + context[:html_title] = title | |
| 177 | + pipeline.call(html, context, data) | |
| 175 | 178 | data |
| 176 | 179 | end |
| 177 | 180 | |
| @@ -180,7 +183,8 @@ module Docs · 2 unmodified lines | ||
| 180 | 183 | end |
| 181 | 184 | |
| 182 | 185 | def parse(string) |
| 183 | − Parser.new(string).html | |
| 186 | + parser = Parser.new(string) | |
| 187 | + [parser.html, parser.title] | |
| 184 | 188 | end |
| 185 | 189 | |
| 186 | 190 | def with_filters(*filters) |
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | module Docs |
| 2 | 2 | class CleanHtmlFilter < Filter |
| 3 | 3 | def call |
| 4 | − css('script', 'style').remove | |
| 4 | + css('script', 'style', 'link').remove | |
| 5 | 5 | xpath('descendant::comment()').remove |
| 6 | 6 | xpath('./text()', './/text()[not(ancestor::pre) and not(ancestor::code) and not(ancestor::div[contains(concat(" ", normalize-space(@class), " "), " prism ")])]').each do |node| |
| 7 | 7 | content = node.content |
| @@ -15,7 +15,7 @@ module Docs · 14 unmodified lines | ||
| 15 | 15 | elsif slug == 'git' || slug.start_with?('git-') |
| 16 | 16 | 'Git' |
| 17 | 17 | else |
| 18 | − 'Miscellaenous' | |
| 18 | + 'Miscellaneous' | |
| 19 | 19 | end |
| 20 | 20 | end |
| 21 | 21 | end |
| @@ -0,0 +1,92 @@ | ||
| 1 | +module Docs | |
| 2 | + class Sqlite | |
| 3 | + class CleanHtmlFilter < Filter | |
| 4 | + def call | |
| 5 | + at_css('.nosearch').remove | |
| 6 | + | |
| 7 | + css('.rightsidebar', 'hr', '.sh_mark', '.fancy_toc > a', '.fancy_toc_mark', 'h[style*="none"]', | |
| 8 | + 'a[href$="intro.html"] > h2', 'a[href$="intro"] > h2', '#document_title + #toc_header', | |
| 9 | + '#document_title ~ #toc').remove | |
| 10 | + | |
| 11 | + css('.fancy_title', '> h2[align=center]', '#document_title').each do |node| | |
| 12 | + node.name = 'h1' | |
| 13 | + end | |
| 14 | + | |
| 15 | + unless at_css('h1') | |
| 16 | + if at_css('h2').content == context[:html_title] | |
| 17 | + at_css('h2').name = 'h1' | |
| 18 | + else | |
| 19 | + doc.child.before("<h1>#{context[:html_title]}</h1>") | |
| 20 | + end | |
| 21 | + end | |
| 22 | + | |
| 23 | + if root_page? | |
| 24 | + at_css('h1').content = 'SQLite Documentation' | |
| 25 | + end | |
| 26 | + | |
| 27 | + css('.codeblock', '.fancy', '.nosearch', '.optab > blockquote', '.optab').each do |node| | |
| 28 | + node.before(node.children).remove | |
| 29 | + end | |
| 30 | + | |
| 31 | + css('blockquote').each do |node| | |
| 32 | + next unless node.at_css('b') | |
| 33 | + node.name = 'pre' | |
| 34 | + node.inner_html = node.inner_html.gsub('<br>', "\n") | |
| 35 | + end | |
| 36 | + | |
| 37 | + css('.todo').each do |node| | |
| 38 | + node.inner_html = "TODO: #{node.inner_html}" | |
| 39 | + end | |
| 40 | + | |
| 41 | + css('blockquote > pre', 'a > h1', 'a > h2', 'center > table', 'ul > ul').each do |node| | |
| 42 | + node.parent.before(node.parent.children).remove | |
| 43 | + end | |
| 44 | + | |
| 45 | + css('table > tr:first-child:last-child > td:first-child:last-child > pre', | |
| 46 | + 'table > tr:first-child:last-child > td:first-child:last-child > ul').each do |node| | |
| 47 | + node.ancestors('table').first.replace(node) | |
| 48 | + end | |
| 49 | + | |
| 50 | + css('a[name]').each do |node| | |
| 51 | + if node.next_element | |
| 52 | + if node.next_element['id'] | |
| 53 | + node.next_element.next_element['id'] ||= node['name'] | |
| 54 | + else | |
| 55 | + node.next_element['id'] = node['name'] | |
| 56 | + end | |
| 57 | + else | |
| 58 | + node.parent['id'] ||= node['name'] | |
| 59 | + end | |
| 60 | + node.remove | |
| 61 | + end | |
| 62 | + | |
| 63 | + unless at_css('h2') | |
| 64 | + css('h1 ~ h1').each do |node| | |
| 65 | + node.name = 'h2' | |
| 66 | + end | |
| 67 | + end | |
| 68 | + | |
| 69 | + css('tt').each do |node| | |
| 70 | + node.name = 'code' | |
| 71 | + end | |
| 72 | + | |
| 73 | + css('pre').each do |node| | |
| 74 | + node.content = node.content | |
| 75 | + node['data-language'] = 'sql' | |
| 76 | + end | |
| 77 | + | |
| 78 | + css('button[onclick]').each do |node| | |
| 79 | + node['class'] = '_btn' | |
| 80 | + node['data-toggle'] = node['onclick'][/hideorshow\("\w+","(\w+)"\)/, 1] | |
| 81 | + node.remove_attribute('onclick') | |
| 82 | + end | |
| 83 | + | |
| 84 | + css('*[align]').remove_attr('align') | |
| 85 | + css('*[style]:not(.imgcontainer)').remove_attr('style') | |
| 86 | + css('table[border]').remove_attr('border') | |
| 87 | + | |
| 88 | + doc | |
| 89 | + end | |
| 90 | + end | |
| 91 | + end | |
| 92 | +end | |
| @@ -0,0 +1,42 @@ | ||
| 1 | +module Docs | |
| 2 | + class Sqlite | |
| 3 | + class CleanJsTablesFilter < Filter | |
| 4 | + def call | |
| 5 | + css('table[id]:empty + script').each do |node| | |
| 6 | + json_list = JSON.parse(node.inner_html[/\[.+?\]/m]) | |
| 7 | + list = '<ul>' | |
| 8 | + json_list.each do |item| | |
| 9 | + list << '<li>' | |
| 10 | + | |
| 11 | + unless item['u'].blank? || item['s'] == 2 | |
| 12 | + list << %(<a href="#{item['u']}">) | |
| 13 | + link = true | |
| 14 | + end | |
| 15 | + | |
| 16 | + if item['s'] == 2 || item['s'] == 3 | |
| 17 | + list << "<s>#{item['x']}</s>" | |
| 18 | + else | |
| 19 | + list << item['x'] | |
| 20 | + end | |
| 21 | + | |
| 22 | + list << '</a>' if link | |
| 23 | + | |
| 24 | + if item['s'] == 1 | |
| 25 | + list << ' <small><i>(exp)</i></small>' | |
| 26 | + elsif item['s'] == 3 | |
| 27 | + list << '¹' | |
| 28 | + elsif item['s'] == 4 | |
| 29 | + list << '²' | |
| 30 | + elsif item['s'] == 5 | |
| 31 | + list << '³' | |
| 32 | + end | |
| 33 | + end | |
| 34 | + list << '</ul>' | |
| 35 | + node.previous_element.replace(list) | |
| 36 | + end | |
| 37 | + | |
| 38 | + doc | |
| 39 | + end | |
| 40 | + end | |
| 41 | + end | |
| 42 | +end | |
| @@ -0,0 +1,79 @@ | ||
| 1 | +module Docs | |
| 2 | + class Sqlite | |
| 3 | + class EntriesFilter < Docs::EntriesFilter | |
| 4 | + ADDITIONAL_ENTRIES = {} | |
| 5 | + | |
| 6 | + def get_name | |
| 7 | + name = context[:html_title] | |
| 8 | + name.remove! 'SQLite Query Language: ' | |
| 9 | + name.remove! %r{\.\z} | |
| 10 | + name = at_css('#document_title').content if name == 'No Title' | |
| 11 | + name | |
| 12 | + end | |
| 13 | + | |
| 14 | + TYPE_BY_SUBPATH_STARTS_WITH = { | |
| 15 | + 'c3ref' => 'C Interface', | |
| 16 | + 'capi' => 'C Interface', | |
| 17 | + 'session' => 'C Interface: Session Module', | |
| 18 | + 'optoverview' => 'Query Planner', | |
| 19 | + 'queryplanner' => 'Query Planner', | |
| 20 | + 'syntax' => 'Syntax Diagrams', | |
| 21 | + 'lang' => 'Language', | |
| 22 | + 'pragma' => 'PRAGMA Statements', | |
| 23 | + 'cli' => 'CLI', | |
| 24 | + 'json' => 'JSON', | |
| 25 | + 'fileformat' => 'Database File Format', | |
| 26 | + 'tcl' => 'Tcl Interface', | |
| 27 | + 'malloc' => 'Dynamic Memory Allocation', | |
| 28 | + 'vtab' => 'Virtual Table Mechanism', | |
| 29 | + 'datatype' => 'Datatypes', | |
| 30 | + 'locking' => 'Locking and Concurrency', | |
| 31 | + 'foreignkey' => 'Foreign Key Constraints', | |
| 32 | + 'wal' => 'Write-Ahead Logging', | |
| 33 | + 'fts' => 'Full-Text Search', | |
| 34 | + 'rtree' => 'R*Tree Module', | |
| 35 | + 'rbu' => 'RBU Extension', | |
| 36 | + 'limits' => 'Limits', | |
| 37 | + 'howtocorrupt' => 'How To Corrupt' | |
| 38 | + } | |
| 39 | + | |
| 40 | + def get_type | |
| 41 | + TYPE_BY_SUBPATH_STARTS_WITH.each_pair do |key, value| | |
| 42 | + return value if subpath.start_with?(key) | |
| 43 | + end | |
| 44 | + | |
| 45 | + if slug.in?(%w(cintro carray c_interface)) | |
| 46 | + 'C Interface' | |
| 47 | + elsif context[:html_title].start_with?('SQLite Query Language') | |
| 48 | + 'Query Language' | |
| 49 | + else | |
| 50 | + 'Miscellaneous' | |
| 51 | + end | |
| 52 | + end | |
| 53 | + | |
| 54 | + IGNORE_ADDITIONAL_ENTRIES_SLUGS = %w(testing uri getthecode whentouse compile) | |
| 55 | + | |
| 56 | + def additional_entries | |
| 57 | + if subpath == 'keyword_index.html' | |
| 58 | + css('li a[href*="#"]').each do |node| | |
| 59 | + slug, id = node['href'].split('#') | |
| 60 | + name = node.content.strip | |
| 61 | + next if name.start_with?('-') || IGNORE_ADDITIONAL_ENTRIES_SLUGS.include?(slug) | |
| 62 | + name.remove! %r{\Athe } | |
| 63 | + name.remove! %r{ SQL function\z} | |
| 64 | + name = 'ORDER BY' if name == 'order by' | |
| 65 | + name.sub!(/\A([a-z])([a-z]+) /) { "#{$1.upcase}#{$2} " } | |
| 66 | + ADDITIONAL_ENTRIES[slug] ||= [] | |
| 67 | + ADDITIONAL_ENTRIES[slug] << [name, id] | |
| 68 | + end | |
| 69 | + end | |
| 70 | + | |
| 71 | + ADDITIONAL_ENTRIES[slug] || [] | |
| 72 | + end | |
| 73 | + | |
| 74 | + def include_default_entry? | |
| 75 | + subpath != 'keyword_index.html' && subpath != 'sitemap.html' | |
| 76 | + end | |
| 77 | + end | |
| 78 | + end | |
| 79 | +end | |
| @@ -0,0 +1,52 @@ | ||
| 1 | +module Docs | |
| 2 | + class Sqlite < FileScraper | |
| 3 | + self.name = 'SQLite' | |
| 4 | + self.type = 'sqlite' | |
| 5 | + self.release = '3.15.2' | |
| 6 | + self.dir = '/Users/Thibaut/DevDocs/Docs/sqlite/' | |
| 7 | + self.base_url = 'https://sqlite.org/' | |
| 8 | + self.root_path = 'docs.html' | |
| 9 | + self.initial_paths = %w(keyword_index.html) | |
| 10 | + self.links = { | |
| 11 | + home: 'https://sqlite.org/', | |
| 12 | + code: 'https://www.sqlite.org/src/' | |
| 13 | + } | |
| 14 | + | |
| 15 | + html_filters.insert_before 'clean_html', 'sqlite/clean_js_tables' | |
| 16 | + html_filters.push 'sqlite/entries', 'sqlite/clean_html' | |
| 17 | + | |
| 18 | + options[:only_patterns] = [/\.html\z/] | |
| 19 | + options[:skip_patterns] = [/releaselog/, /consortium/] | |
| 20 | + options[:skip] = %w( | |
| 21 | + index.html | |
| 22 | + about.html | |
| 23 | + download.html | |
| 24 | + copyright.html | |
| 25 | + support.html | |
| 26 | + prosupport.html | |
| 27 | + hp1.html | |
| 28 | + news.html | |
| 29 | + oldnews.html | |
| 30 | + doclist.html | |
| 31 | + dev.html | |
| 32 | + chronology.html | |
| 33 | + not-found.html | |
| 34 | + famous.html | |
| 35 | + books.html | |
| 36 | + crew.html | |
| 37 | + mostdeployed.html | |
| 38 | + requirements.html | |
| 39 | + session/intro.html | |
| 40 | + syntax.html | |
| 41 | + ) | |
| 42 | + | |
| 43 | + options[:attribution] = 'SQLite is in the Public Domain.' | |
| 44 | + | |
| 45 | + private | |
| 46 | + | |
| 47 | + def parse(html) | |
| 48 | + html.gsub! %r{(<h2[^>]*>[^<]+)</h1>}, '\1</h2>' | |
| 49 | + super | |
| 50 | + end | |
| 51 | + end | |
| 52 | +end | |