History

Finish GNU Fortran scraper

Thibaut Courouble committed 10 years ago

Commit 6b37efd1 parent d366e1417 files · +123 −81Browse files

Showing the changes to lib/docs (6 of 17 files). Show all

Unified · Split

6 files
  1. lib/docs/filters/fortran/clean_html.rb +0 −18
  2. lib/docs/filters/fortran/entries.rb +0 −34
  3. lib/docs/filters/gnu_fortran/clean_html.rb +39 −0
  4. lib/docs/filters/gnu_fortran/entries.rb +41 −0
  5. lib/docs/scrapers/fortran.rb +0 −32
  6. lib/docs/scrapers/gnu_fortran.rb +40 −0
lib/docs/filters/fortran/clean_html.rb+0−18
@@ -1,18 +0,0 @@
1−module Docs
2− class Fortran
3− class CleanHtmlFilter < Filter
4− def call
5− css('h2', 'h3', 'h4').each do |node|
6− node.name = 'h1'
7− end
8−
9− # Move page anchor to page title
10− at_css('h1')['id'] = at_css('.node > a')['name']
11−
12− css('.node', 'br').remove
13−
14− doc
15− end
16− end
17− end
18−end
lib/docs/filters/fortran/entries.rb+0−34
@@ -1,34 +0,0 @@
1−module Docs
2− class Fortran
3− class EntriesFilter < Docs::EntriesFilter
4− REPLACE_TYPES = {
5− 1 => 'Introduction',
6− 2 => 'GNU Fortran Command Options',
7− 3 => 'Runtime Environment Variables',
8− 4 => 'Fortran 2003 and 2008 Status',
9− 5 => 'Compiler Characteristics',
10− 6 => 'Extensions',
11− 7 => 'Mixed Language Programming',
12− 8 => 'Coarray Programming',
13− 9 => 'Intrinsic Procedures',
14− 10 => 'Intrinsic Modules' }
15−
16− def chapter_number
17− at_css('h1').content.to_i
18− end
19−
20− def include_default_entry?
21− REPLACE_TYPES[chapter_number] and not at_css('ul.menu')
22− end
23−
24− def get_name
25− at_css('h1').content.split(' ').drop(1).join(' ').split('—').first
26− end
27−
28− def get_type
29− REPLACE_TYPES[chapter_number]
30− end
31−
32− end
33− end
34−end
lib/docs/filters/gnu_fortran/clean_html.rb+39−0
@@ -0,0 +1,39 @@
1+module Docs
2+ class GnuFortran
3+ class CleanHtmlFilter < Filter
4+ def call
5+ heading = at_css('h1, h2, h3, h4, h5')
6+ heading_level = heading.name[/h(\d)/, 1].to_i
7+
8+ css('h2, h3, h4, h5, h6').each do |node|
9+ node.name = node.name.sub(/\d/) { |i| i.to_i - (heading_level - 1) }
10+ end
11+
12+ css('.node > a[name]').each do |node|
13+ node.parent.next_element['id'] = node['name']
14+ end
15+
16+ css('a[name]').each do |node|
17+ node['id'] = node['name']
18+ end
19+
20+ css('samp > span:first-child:last-child').each do |node|
21+ node.parent.name = 'code'
22+ node.before(node.children).remove
23+ end
24+
25+ css('pre').each do |node|
26+ node.inner_html = node.inner_html.strip_heredoc.strip
27+ end
28+
29+ css('dt > em', 'acronym', 'dfn').each do |node|
30+ node.before(node.children).remove
31+ end
32+
33+ css('.node', 'br').remove
34+
35+ doc
36+ end
37+ end
38+ end
39+end
lib/docs/filters/gnu_fortran/entries.rb+41−0
@@ -0,0 +1,41 @@
1+module Docs
2+ class GnuFortran
3+ class EntriesFilter < Docs::EntriesFilter
4+ TYPE_BY_CHAPTER = { }
5+
6+ def initialize(*)
7+ super
8+ detect_chapters if root_page?
9+ end
10+
11+ def get_name
12+ at_css('h1').content.split(' ').drop(1).join(' ').split('—').first
13+ end
14+
15+ def get_type
16+ "#{chapter_number}. #{TYPE_BY_CHAPTER[chapter_number]}"
17+ end
18+
19+ def include_default_entry?
20+ !at_css('ul.menu')
21+ end
22+
23+ private
24+
25+ def detect_chapters
26+ css('.contents > ul > li > a').each do |node|
27+ index = node.content.strip.to_i
28+ next unless index > 0
29+ name = node.content.split(' ').drop(1).join(' ')
30+ name.remove! 'GNU Fortran '
31+ name.remove! %r{:.*}
32+ TYPE_BY_CHAPTER[index] = name # YOLO
33+ end
34+ end
35+
36+ def chapter_number
37+ at_css('h1').content.to_i
38+ end
39+ end
40+ end
41+end
lib/docs/scrapers/fortran.rb+0−32
@@ -1,32 +0,0 @@
1−module Docs
2− class Fortran < FileScraper
3− self.name = 'GNU Fortran'
4− self.slug = 'fortran'
5− self.type = 'fortran'
6− self.release = '5.3.0'
7− self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
8− self.dir = ''
9− self.root_path = 'index.html'
10−
11− self.links = {
12− home: 'https://gcc.gnu.org/fortran/'
13− }
14−
15− html_filters.push 'fortran/clean_html', 'fortran/entries'
16−
17− options[:skip_patterns] = [
18− /Funding/,
19− /Projects/,
20− /Copying/,
21− /License/,
22− /Proposed/,
23− /Contribut/,
24− /Index/
25− ]
26−
27− options[:attribution] = <<-HTML
28− &copy; Free Software Foundation<br>
29− Licensed under the GNU Free Documentation License version 1.3.
30− HTML
31− end
32−end
lib/docs/scrapers/gnu_fortran.rb+40−0
@@ -0,0 +1,40 @@
1+module Docs
2+ class GnuFortran < FileScraper
3+ self.name = 'GNU Fortran'
4+ self.slug = 'gnu_fortran'
5+ self.type = 'gnu_fortran'
6+ self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran'
7+ self.root_path = 'index.html'
8+
9+ self.links = {
10+ home: 'https://gcc.gnu.org/fortran/'
11+ }
12+
13+ html_filters.push 'gnu_fortran/clean_html', 'gnu_fortran/entries'
14+
15+ options[:skip_patterns] = [
16+ /Funding/,
17+ /Projects/,
18+ /Copying/,
19+ /License/,
20+ /Proposed/,
21+ /Contribut/,
22+ /Index/
23+ ]
24+
25+ options[:attribution] = <<-HTML
26+ &copy; Free Software Foundation<br>
27+ Licensed under the GNU Free Documentation License, Version 1.3.
28+ HTML
29+
30+ version '5' do
31+ self.release = '5.3.0'
32+ self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
33+ end
34+
35+ version '4' do
36+ self.release = '4.9.3'
37+ self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
38+ end
39+ end
40+end