Finish GNU Fortran scraper

Thibaut Courouble committed 10 years ago

Commit 6b37efd1 parent d366e1417 files · +123 −81Browse files

Showing the changes to lib/docs (6 of 17 files). Show all

Unified · Split

6 files

@@ -1,18 +0,0 @@
1	module Docs
2	class Fortran
3	class CleanHtmlFilter < Filter
4	def call
5	css('h2', 'h3', 'h4').each do \|node\|
6	node.name = 'h1'
7	end
8
9	# Move page anchor to page title
10	at_css('h1')['id'] = at_css('.node > a')['name']
11
12	css('.node', 'br').remove
13
14	doc
15	end
16	end
17	end
18	end

@@ -1,34 +0,0 @@
1	module Docs
2	class Fortran
3	class EntriesFilter < Docs::EntriesFilter
4	REPLACE_TYPES = {
5	1 => 'Introduction',
6	2 => 'GNU Fortran Command Options',
7	3 => 'Runtime Environment Variables',
8	4 => 'Fortran 2003 and 2008 Status',
9	5 => 'Compiler Characteristics',
10	6 => 'Extensions',
11	7 => 'Mixed Language Programming',
12	8 => 'Coarray Programming',
13	9 => 'Intrinsic Procedures',
14	10 => 'Intrinsic Modules' }
15
16	def chapter_number
17	at_css('h1').content.to_i
18	end
19
20	def include_default_entry?
21	REPLACE_TYPES[chapter_number] and not at_css('ul.menu')
22	end
23
24	def get_name
25	at_css('h1').content.split(' ').drop(1).join(' ').split('—').first
26	end
27
28	def get_type
29	REPLACE_TYPES[chapter_number]
30	end
31
32	end
33	end
34	end

@@ -0,0 +1,39 @@
	1	module Docs
	2	class GnuFortran
	3	class CleanHtmlFilter < Filter
	4	def call
	5	heading = at_css('h1, h2, h3, h4, h5')
	6	heading_level = heading.name[/h(\d)/, 1].to_i
	7
	8	css('h2, h3, h4, h5, h6').each do \|node\|
	9	node.name = node.name.sub(/\d/) { \|i\| i.to_i - (heading_level - 1) }
	10	end
	11
	12	css('.node > a[name]').each do \|node\|
	13	node.parent.next_element['id'] = node['name']
	14	end
	15
	16	css('a[name]').each do \|node\|
	17	node['id'] = node['name']
	18	end
	19
	20	css('samp > span:first-child:last-child').each do \|node\|
	21	node.parent.name = 'code'
	22	node.before(node.children).remove
	23	end
	24
	25	css('pre').each do \|node\|
	26	node.inner_html = node.inner_html.strip_heredoc.strip
	27	end
	28
	29	css('dt > em', 'acronym', 'dfn').each do \|node\|
	30	node.before(node.children).remove
	31	end
	32
	33	css('.node', 'br').remove
	34
	35	doc
	36	end
	37	end
	38	end
	39	end

@@ -0,0 +1,41 @@
	1	module Docs
	2	class GnuFortran
	3	class EntriesFilter < Docs::EntriesFilter
	4	TYPE_BY_CHAPTER = { }
	5
	6	def initialize(*)
	7	super
	8	detect_chapters if root_page?
	9	end
	10
	11	def get_name
	12	at_css('h1').content.split(' ').drop(1).join(' ').split('—').first
	13	end
	14
	15	def get_type
	16	"#{chapter_number}. #{TYPE_BY_CHAPTER[chapter_number]}"
	17	end
	18
	19	def include_default_entry?
	20	!at_css('ul.menu')
	21	end
	22
	23	private
	24
	25	def detect_chapters
	26	css('.contents > ul > li > a').each do \|node\|
	27	index = node.content.strip.to_i
	28	next unless index > 0
	29	name = node.content.split(' ').drop(1).join(' ')
	30	name.remove! 'GNU Fortran '
	31	name.remove! %r{:.*}
	32	TYPE_BY_CHAPTER[index] = name # YOLO
	33	end
	34	end
	35
	36	def chapter_number
	37	at_css('h1').content.to_i
	38	end
	39	end
	40	end
	41	end

@@ -1,32 +0,0 @@
1	module Docs
2	class Fortran < FileScraper
3	self.name = 'GNU Fortran'
4	self.slug = 'fortran'
5	self.type = 'fortran'
6	self.release = '5.3.0'
7	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
8	self.dir = ''
9	self.root_path = 'index.html'
10
11	self.links = {
12	home: 'https://gcc.gnu.org/fortran/'
13	}
14
15	html_filters.push 'fortran/clean_html', 'fortran/entries'
16
17	options[:skip_patterns] = [
18	/Funding/,
19	/Projects/,
20	/Copying/,
21	/License/,
22	/Proposed/,
23	/Contribut/,
24	/Index/
25	]
26
27	options[:attribution] = <<-HTML
28	© Free Software Foundation<br>
29	Licensed under the GNU Free Documentation License version 1.3.
30	HTML
31	end
32	end

@@ -0,0 +1,40 @@
	1	module Docs
	2	class GnuFortran < FileScraper
	3	self.name = 'GNU Fortran'
	4	self.slug = 'gnu_fortran'
	5	self.type = 'gnu_fortran'
	6	self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran'
	7	self.root_path = 'index.html'
	8
	9	self.links = {
	10	home: 'https://gcc.gnu.org/fortran/'
	11	}
	12
	13	html_filters.push 'gnu_fortran/clean_html', 'gnu_fortran/entries'
	14
	15	options[:skip_patterns] = [
	16	/Funding/,
	17	/Projects/,
	18	/Copying/,
	19	/License/,
	20	/Proposed/,
	21	/Contribut/,
	22	/Index/
	23	]
	24
	25	options[:attribution] = <<-HTML
	26	© Free Software Foundation<br>
	27	Licensed under the GNU Free Documentation License, Version 1.3.
	28	HTML
	29
	30	version '5' do
	31	self.release = '5.3.0'
	32	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
	33	end
	34
	35	version '4' do
	36	self.release = '4.9.3'
	37	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
	38	end
	39	end
	40	end