Improve GitHub scraper

Thibaut Courouble committed 9 years ago

Commit 85fffdc1 parent 80cc88c4 files · +10 −10Browse files

Unified · Split

4 files

@@ -1,9 +0,0 @@
1	#= require views/pages/base
2
3	class app.views.GithubPage extends app.views.BasePage
4	LANGUAGE_RGX = /highlight-source-(\w+)/
5
6	prepare: ->
7	for el in @findAll('pre.highlight')
8	@highlightCode(el, el.className.match(LANGUAGE_RGX)[1])
9	return

@@ -28,6 +28,7 @@ app.views.EmberPage = · 27 unmodified lines
28	app.views.ErlangPage =	28	app.views.ErlangPage =
29	app.views.ExpressPage =	29	app.views.ExpressPage =
30	app.views.FlowPage =	30	app.views.FlowPage =
		31	app.views.GithubPage =
31	app.views.GoPage =	32	app.views.GoPage =
32	app.views.ImmutablePage =	33	app.views.ImmutablePage =
33	app.views.InfluxdataPage =	34	app.views.InfluxdataPage =

         end
         css('.highlight > pre').each do |node|
-          node['class'] = node.parent['class']
+          node['data-language'] = node.parent['class'][/highlight-source-(\w+)/, 1]
           node.content = node.content.strip_heredoc.gsub('    ', '  ')
           node.parent.replace(node)
         end
+        css('pre > code').each do |node|
+          node.before(node.children).remove
+        end
         doc
       end
     end

           node.name = node.name.sub(/\d/) { |i| i.to_i + 1 }
         end
+        css('a > img').each do |node|
+          node.parent.before(node.parent.children).remove
+        end
         doc
       end
     end