Finish Phalcon scraper

Thibaut committed 11 years ago

Commit ece7a751 parent 4cde9d214 files · +69 −19Browse files

Unified · Split

14 files

Binary file not shown.

Binary file not shown.

 [
   [
+    "2015-09-13",
+    "New documentation: <a href=\"/phalcon/\">Phalcon</a>"
+  ], [
     "2015-08-09",
     "New documentation: <a href=\"/react_native/\">React Native</a>"
   ], [

     '2010-2015 The OpenTSDB Authors',
     'LGPLv2.1',
     'https://raw.githubusercontent.com/OpenTSDB/opentsdb.net/gh-pages/COPYING.LESSER'
+  ], [
+    'Phalcon',
+    '2011-2015 Phalcon Framework Team',
+    'CC BY',
+    'https://docs.phalconphp.com/en/latest/reference/license.html'
   ], [
     'Phaser',
     '2015 Richard Davey, Photon Storm Ltd.',

@@ -0,0 +1,7 @@
	1	#= require views/pages/base
	2
	3	class app.views.PhalconPage extends app.views.BasePage
	4	afterRender: ->
	5	@highlightCode @findAll('pre[class*="php"]'), 'php'
	6	@highlightCode @findAll('pre.highlight-html'), 'markup'
	7	return

@@ -112,3 +112,4 @@ · 111 unmodified lines
112	%icon-expand-white { background-position: -8rem -8rem; }	112	%icon-expand-white { background-position: -8rem -8rem; }
113	%icon-contract-white { background-position: -9rem -8rem; }	113	%icon-contract-white { background-position: -9rem -8rem; }
114	._icon-react_native:before { background-position: 0 -9rem; }	114	._icon-react_native:before { background-position: 0 -9rem; }
		115	._icon-phalcon:before { background-position: -1rem -9rem; }

@@ -4,12 +4,38 @@ module Docs · 3 unmodified lines
4	def call	4	def call
5	@doc = at_css('.body')	5	@doc = at_css('.body')
6		6
7	# Remove unnecessary things	7	if root_page?
8	css('.headerlink', '#what-is-phalcon', '#other-formats', '#welcome h1', '#welcome p', '#table-of-contents h2').remove	8	at_css('h1').content = 'Phalcon'
		9	end
		10
		11	css('#what-is-phalcon', '#other-formats').remove
		12
		13	css('#methods > p > strong, #constants > p > strong').each do \|node\|
		14	node.parent.name = 'h3'
		15	node.parent['id'] = node.content.parameterize
		16	node.parent['class'] = 'method-signature'
		17	node.parent.inner_html = node.parent.inner_html.sub(/inherited from .*/, '<small>\0</small>')
		18	end
		19
		20	css('.headerlink').each do \|node\|
		21	id = node['href'][1..-1]
		22	node.parent['id'] \|\|= id
		23	node.remove
		24	end
		25
		26	css('div[class^="highlight-"]').each do \|node\|
		27	code = node.at_css('pre').content
		28	code.remove! %r{\A\s<\?php\s} unless code.include?(' ?>')
		29	node.content = code
		30	node.name = 'pre'
		31	end
		32
		33	css('.section').each do \|node\|
		34	node.before(node.children).remove
		35	end
9		36
10	# Add id for constants and methods	37	css('table[border]').each do \|node\|
11	css('#constants strong', '#methods strong').each do \|node\|	38	node.remove_attribute('border')
12	node.parent['id'] = node.content.strip
13	end	39	end
14		40
15	doc	41	doc

@@ -1,31 +1,28 @@
1	module Docs	1	module Docs
2	class Phalcon	2	class Phalcon
3	class EntriesFilter < Docs::EntriesFilter	3	class EntriesFilter < Docs::EntriesFilter
4
5	def get_name	4	def get_name
6	node = css('h1').first	5	(at_css('h1 > strong') \|\| at_css('h1')).content.strip.remove('Phalcon\\')
7	name = node.content.strip
8	node.remove
9	name
10	end	6	end
11		7
12	def get_type	8	def get_type
13	if slug.start_with? 'reference'	9	if slug.start_with?('reference')
14	'Guides'	10	'Guides'
15	else	11	else
16	'Classes'	12	path = name.split('\\')
		13	path[1] == 'Mvc' ? path[1..2].join('\\') : path[1]
17	end	14	end
18	end	15	end
19		16
20	def additional_entries	17	def additional_entries
21	entries = []	18	entries = []
22		19
23	css('#constants strong').each do \|node\|	20	css('.method-signature').each do \|node\|
24	entries << [node.content.strip, node.parent['id'], 'Constants']	21	next if node.content.include?('inherited from') \|\| node.content.include?('protected ') \|\| node.content.include?('private ')
25	end	22	name = node.at_css('strong').content.strip
26		23	next if name == '__construct' \|\| name == '__toString'
27	css('#methods strong').each do \|node\|	24	name.prepend "#{self.name}::"
28	entries << [node.content.strip, node.parent['id'], 'Methods']	25	entries << [name, node['id']]
29	end	26	end
30		27
31	entries	28	entries

 module Docs
   class Phalcon < UrlScraper
-    self.name = 'Phalcon'
-    self.slug = 'phalcon'
     self.type = 'phalcon'
+    self.version = '2.0.6'
     self.base_url = 'https://docs.phalconphp.com/en/latest/'
     self.root_path = 'index.html'
     self.links = {
       code: 'https://github.com/phalcon/cphalcon/'
     }
-    html_filters.push 'phalcon/clean_html', 'phalcon/entries', 'title'
+    html_filters.push 'phalcon/clean_html', 'phalcon/entries'
     options[:root_title] = 'Phalcon'
     options[:only_patterns] = [/reference\//, /api\//]
-    options[:skip_patterns] = [/api\/index/]
+    options[:skip] = %w(
+      api/index.html
+      reference/license.html)
     options[:attribution] = <<-HTML
-      &copy; 2012&ndash;2015 the Phalcon Team<br>
+      &copy; 2011&ndash;2015 Phalcon Framework Team<br>
       Licensed under the Creative Commons Attribution License 3.0.
     HTML
   end

Binary file not shown.

Binary file not shown.