History

Finish Docker scraper

Thibaut Courouble committed 10 years ago

Commit e92d0e11 parent 457439f11 files · +65 −21Browse files

Showing the changes to lib/docs (3 of 11 files). Show all

Unified · Split

3 files
  1. lib/docs/filters/docker/clean_html.rb +5 −6
  2. lib/docs/filters/docker/entries.rb +44 −8
  3. lib/docs/scrapers/docker.rb +10 −10
lib/docs/filters/docker/clean_html.rb+5−6
@@ -3,14 +3,13 @@ module Docs · 2 unmodified lines
33 class CleanHtmlFilter < Filter
44 def call
55 if root_page?
6− @doc = at_css('.tabs-content')
7− else
8− @doc = at_css('#content')
6+ doc.inner_html = "<h1>Docker Documentation</h1>"
7+ return doc
98 end
109
11− if not at_css('h1')
12− at_css('h2').name = 'h1'
13− end
10+ @doc = at_css('#content')
11+
12+ at_css('h2').name = 'h1' unless at_css('h1')
1413
1514 css('pre').each do |node|
1615 node.content = node.content
lib/docs/filters/docker/entries.rb+44−8
@@ -2,20 +2,56 @@ module Docs · 1 unmodified line
22 class Docker
33 class EntriesFilter < Docs::EntriesFilter
44 def get_name
5− title = at_css('h1').content
5+ name = nav_link ? nav_link.content.strip : at_css('#content h1').content.strip
6+ name.capitalize! if name == 'exoscale'
67
7− title = title.split(' — ').first
8− title = title.split(' - ').first
8+ if name =~ /\A[a-z\-\s]+\z/
9+ name.prepend 'docker ' if subpath =~ /engine\/reference\/commandline\/./
10+ name.prepend 'docker-compose ' if subpath =~ /compose\/reference\/./
11+ name.prepend 'docker-machine ' if subpath =~ /machine\/reference\/./
12+ name.prepend 'swarm ' if subpath =~ /swarm\/reference\/./ && name != 'swarm'
13+ else
14+ name << " (#{product})" if name !~ /#{product}/i
15+ end
916
10− title
17+ name
1118 end
1219
1320 def get_type
14− if slug.count('/') == 1 or slug.count('/') == 2
15− slug.split('/').first.capitalize
16− else
17− slug.split('/')[0...-1].map(&:capitalize).join('/').gsub(/\-|\_/, ' ')
21+ unless nav_link
22+ return 'Engine: User guide' if subpath.start_with?('engine/userguide')
23+ return 'Engine: Secure' if subpath.start_with?('engine/security')
24+ return 'Engine' if subpath == 'engine/'
25+ end
26+
27+ type = nav_link.ancestors('article').to_a.reverse.to_a[0..1].map do |node|
28+ node.at_css('> button').content.strip
29+ end.join(': ')
30+
31+ type.remove! %r{\ADocker }
32+ type.remove! %r{ Engine}
33+ type.sub! %r{Command[\-\s]line reference}i, 'CLI'
34+ type = 'Engine: Reference' if type == 'Engine: reference'
35+ type
36+ end
37+
38+ def nav_link
39+ return @nav_link if defined?(@nav_link)
40+ @nav_link = at_css('#multiple .active')
41+
42+ unless @nav_link
43+ link = at_css('#content li a')
44+ return unless link
45+ link = at_css("#multiple a[href='#{link['href']}']")
46+ return unless link
47+ @nav_link = link.ancestors('article').first.at_css('button')
1848 end
49+
50+ @nav_link
51+ end
52+
53+ def product
54+ @product ||= subpath.split('/').first.capitalize
1955 end
2056 end
2157 end
lib/docs/scrapers/docker.rb+10−10
@@ -1,16 +1,14 @@
11 module Docs
22 class Docker < UrlScraper
3− self.name = 'Docker'
4− self.slug = 'docker'
53 self.type = 'docker'
6−
74 self.links = {
85 home: 'https://docker.com/',
96 code: 'https://github.com/docker/docker'
107 }
118
12− html_filters.push 'docker/clean_html', 'docker/entries'
9+ html_filters.push 'docker/entries', 'docker/clean_html'
1310
11+ options[:container] = '#docs'
1412 options[:trailing_slash] = true
1513
1614 options[:only_patterns] = [
@@ -19,14 +17,16 @@ module Docs · 2 unmodified lines
1917 /\Amachine\//,
2018 /\Aswarm\//
2119 ]
22−
23− options[:skip] = [
24− "engine/installation/ubuntulinux/" # invalid document
25− ]
20+ options[:skip] = %w(swarm/scheduler/)
21+ options[:replace_paths] = {
22+ 'engine/installation/ubuntulinux/' => 'engine/installation/linux/ubuntulinux/'
23+ }
2624
2725 options[:attribution] = <<-HTML
28− &copy; 2016 Docker Inc.<br>
29− Licensed under the Apache 2 License.
26+ &copy; 2013&ndash;2016 Docker, Inc.<br>
27+ Licensed under the Apache License, Version 2.0.<br>
28+ Docker and the Docker logo are trademarks or registered trademarks of Docker, Inc. in the United States and/or other countries.<br>
29+ Docker, Inc. and other parties may also have trademark rights in other terms used herein.
3030 HTML
3131
3232 version '1.11' do