codescrape

codescrape Commit Details


Date:2015-09-08 21:56:08 (9 years 3 months ago)
Author:Natalie Adams
Branch:master
Commit:88d65df2bb567eea5fdd6e7b9f9b7375672cfb4d
Message:initial commit
Changes:

File differences

Issue.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
class IssueComment(object):␊
␊
␊
author = ""␊
title = ""␊
summary = ""␊
date = ""␊
␊
def getAuthor(self):␊
return self.author␊
␊
def getTitle(self):␊
return self.title␊
␊
def getSummary(self):␊
return self.summary␊
␊
def getDate(self):␊
return self.date␊
␊
␊
␊
class Issue(IssueComment):␊
␊
status = ""␊
comments = []␊
id = -1␊
␊
def getId(self):␊
return self.id␊
␊
def getStatus(self):␊
return self.status␊
␊
def getCommnets(self):␊
return self.comments
Project.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
from enum import enum␊
␊
REPO_TYPES = enum("SVN", "git", "hg", "NA")␊
␊
class Project(object):␊
␊
repoURL = ""␊
releases = []␊
issues = []␊
wikis = []␊
repoType = REPO_TYPES.NA␊
␊
def getRepoURL(self):␊
return self.repoURL␊
␊
def getReleases(self):␊
return self.releases␊
␊
def getIssues(self):␊
return self.issues␊
␊
def getRepoType(self):␊
return self.repoType␊
␊
def getWikis(self):␊
return self.wikis
README.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# codescrape␊
␊
Version 1.0␊
␊
By: Nathan Adams␊
␊
License: MIT␊
␊
## Description␊
␊
This library is to be used to archive project data. Since with the announcement of Google Code going to archive only - I wanted to create a library where you can grab source data before it is gone forever.␊
␊
Use cases include:␊
␊
Archive projects due to:␊
␊
- Hosting service shutting down␊
- Authorities sending cease-and-desist against provider/project␊
- Historical/research/ or educational purposes␊
␊
## Usage␊
␊
Currently srchub and google code are supported. To use:␊
␊
from services.srchub import srchub␊
␉shub = srchub()␊
␉projects = shub.getProjects()␊
␉␊
or for google code␊
␊
␉from services.googlecode import googlecode␊
␉gcode = googlecode()␊
␉project = gcode.getProject("android-python27")␊
␉␊
Sourcehub library will pull all public projects since this list is easily accessed. Google Code does not have a public list persay. And I didn't want to scrape the search results, so I developed it to require you to pass in the project name. If you were to get your hands on a list of google code projects you could easily loop through them:␊
␊
␉from services.googlecode import googlecode␊
␉gcode = googlecode()␊
␉for project in someProjectList:␊
␉␉project = gcode.getProject(project)␊
␉␉# do something with project␊
␉␉␊
the project data structure is as follows:␊
␊
project␊
␊
- getRepoURL() -> Returns the URL of the repo␊
- getRepoType() -> Returns the type of repo (git, hg, or SVN)␊
- getReleases() -> Returns all downloads related to the project␊
- getIssues() -> Returns open issues␊
- getWikis() -> Returns wikis␊
␊
Release.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
class Release(object):␊
␊
fileName = ""␊
summary = ""␊
fileURL = ""␊
checksum = None␊
␊
def getFileName(self):␊
return self.fileName␊
␊
def getSummary(self):␊
return self.summary␊
␊
def getFileURL(self):␊
return self.fileURL␊
␊
def getChecksum(self):␊
return self.checksum
Service.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
import pycurl␊
try:␊
from cStringIO import StringIO␊
except ImportError:␊
try:␊
from StringIO import StringIO␊
except ImportError:␊
from io import StringIO␊
from urllib import urlencode␊
␊
class Service(object):␊
␊
def getProjects(self):␊
pass␊
␊
def curl_post(self, url, postvals, header = []):␊
buffer = StringIO()␊
cobj = pycurl.Curl()␊
cobj.setopt(pycurl.URL, url)␊
cobj.setopt(pycurl.SSL_VERIFYPEER, 0)␊
cobj.setopt(pycurl.SSL_VERIFYHOST, 0)␊
cobj.setopt(pycurl.POST, 1)␊
cobj.setopt(pycurl.WRITEDATA, buffer)␊
postdata = urlencode(postvals)␊
cobj.setopt(pycurl.POSTFIELDS, postdata)␊
cobj.setopt(pycurl.HTTPHEADER, header)␊
cobj.perform()␊
cobj.close()␊
return buffer␊
␊
def curl_get(self, url, header = []):␊
buffer = StringIO()␊
cobj = pycurl.Curl()␊
cobj.setopt(pycurl.SSL_VERIFYPEER, 0)␊
cobj.setopt(pycurl.SSL_VERIFYHOST, 0)␊
cobj.setopt(pycurl.URL, url)␊
cobj.setopt(pycurl.WRITEDATA, buffer)␊
cobj.setopt(pycurl.HTTPHEADER, header)␊
cobj.perform()␊
cobj.close()␊
return buffer
Wiki.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
class Wiki(object):␊
␊
pageName = ""␊
htmlContent = ""␊
textContent = ""␊
summary = ""␊
updated = ""␊
␊
def getPageName(self):␊
return self.pageName␊
␊
def getHTMLCotnent(self):␊
return self.htmlContent␊
␊
def getTextContent(self):␊
return self.textContent␊
␊
def getSummary(self):␊
return self.summary␊
␊
def getUpdated(self):␊
return self.updated
enum.py
1
2
3
4
5
6
7
# Pythonic way to do enums:␊
# http://stackoverflow.com/a/1695250/195722␊
def enum(*sequential, **named):␊
enums = dict(zip(sequential, range(len(sequential))), **named)␊
reverse = dict((value, key) for key, value in enums.iteritems())␊
enums['val'] = reverse␊
return type('Enum', (), enums)
main.py
1
2
3
4
5
6
7
8
from services.srchub import srchub␊
from services.googlecode import googlecode␊
␊
#shub = srchub()␊
␊
#projects = shub.getProjects()␊
gcode = googlecode()␊
project = gcode.getProject("android-python27")␊
services/__init__.py
1
__author__ = 'nathan'␊
services/googlecode.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
from Service import Service␊
from Project import REPO_TYPES, Project␊
from Release import Release␊
from Issue import IssueComment, Issue␊
from Wiki import Wiki␊
from bs4 import BeautifulSoup␊
␊
␊
class googlecode(Service):␊
␊
DOMAIN = "https://code.google.com"␊
␊
# Since I want to stay on Google's good side␊
# I'm going to write this method to parse a single project␊
# You will need to provide your own project list to roll through␊
# Such a list exists (although incomplete)␊
# http://flossdata.syr.edu/data/gc/2012/2012-Nov/gcProjectInfo2012-Nov.txt.bz2␊
def getProject(self, projectName):␊
project = Project()␊
sourceType = None␊
projectURL = self.DOMAIN + "/p/" + projectName + "/"␊
␊
projectpageHTML = self.curl_get(projectURL).getvalue()␊
projectpageSoup = BeautifulSoup(projectpageHTML, "html.parser")␊
␊
sourceURL = projectpageSoup.find(name="a", string="Source").get("href")␊
sourceSoup = BeautifulSoup(self.curl_get(self.DOMAIN + "/p/" + sourceURL).getvalue(), "html.parser")␊
sourceSoupText = sourceSoup.get_text()␊
␊
# get source␊
if "git clone" in sourceSoupText:␊
project.repoType = REPO_TYPES.git␊
project.repoURL = "https://code.google.com/p/" + projectName + "/"␊
elif "svn co" in sourceSoupText:␊
project.repoType = REPO_TYPES.SVN␊
project.repoURL = "http://" + projectName + ".googlecode.com/svn/"␊
else:␊
project.repoType = REPO_TYPES.hg␊
project.repoURL = "https://code.google.com/p/" + projectName + "/"␊
␊
␊
# get downloads␊
project.releases = []␊
downlaodsSoup = BeautifulSoup(self.curl_get(projectURL + "downloads/list").getvalue(), "html.parser")␊
downloadSection = downlaodsSoup.find("table", "results")␊
if "Your search did not generate any results." not in downlaodsSoup.get_text():␊
downloadRows = downloadSection.find_all("tr")[1:]␊
for downloadRow in downloadRows:␊
cols = downloadRow.find_all("td")␊
downloadTD = cols[1]␊
downloadURL = "https://" + projectName + ".googlecode.com/files/" + downloadTD.a.text.replace("\n", "").strip(" ")␊
fileName = downloadTD.a.text.replace("\n", "").strip(" ")␊
release = Release()␊
release.fileURL = downloadURL␊
release.fileName = fileName␊
project.releases.append(release)␊
␊
# get issues␊
project.issues = []␊
issuesSoup = BeautifulSoup(self.curl_get(projectURL + "issues/list").getvalue(), "html.parser")␊
if "Your search did not generate any results." not in issuesSoup.get_text():␊
issuesSection = issuesSoup.find("table", "results")␊
for issueRow in issuesSection.find_all("tr")[1:]:␊
issue = Issue()␊
cols = issueRow.find_all("td")␊
issueId = cols[1].text.replace("\n", "").strip()␊
issueURL = projectURL + "issues/detail?id=" + issueId␊
issueStatus = cols[3].text.replace("\n", "").strip(" ")␊
issueSummary = cols[8].text.replace("\n", "")␊
issueTitle = cols[8].text.replace("\n", "")␊
issueAuthor = cols[5].text.replace("\n", "")␊
␊
#issue.author = issueAuthor␊
issue.comments = []␊
issue.status = issueStatus.strip(" ")␊
issue.summary = issueSummary.strip(" ")␊
issue.title = issueTitle␊
issue.id = issueId␊
␊
# we must go deeper to get comments␊
issueComments = BeautifulSoup(self.curl_get(issueURL).getvalue(), "html.parser")␊
for comment in issueComments.find_all("div", "vt"):␊
#author = comment.find(class_="author").find("a").text␊
author = (comment.find(class_="author").find_all("a")[-1]).contents␊
date = comment.find("span", "date")["title"]␊
commentText = comment.find("pre").get_text()␊
issueComment = IssueComment()␊
issueComment.date = date␊
issueComment.author = author␊
issueComment.summary = commentText␊
issue.comments.append(issueComment)␊
␊
project.issues.append(issue)␊
␊
# get wiki pages␊
project.wikis = []␊
wikiSoup = BeautifulSoup(self.curl_get(projectURL + "w/list").getvalue(), "html.parser")␊
if "Your search did not generate any results." not in wikiSoup.get_text():␊
wikiSection = wikiSoup.find("table", "results")␊
for wikiRow in wikiSection.find_all("tr")[1:]:␊
wiki = Wiki()␊
cols = wikiRow.find_all("td")␊
wiki.pageName = cols[1].text.replace("\n", "").strip(" ")␊
wiki.summary = cols[2].text.replace("\n", "").strip(" ")␊
wiki.updated = cols[3].text.replace("\n", "").strip(" ")␊
wikiURL = projectURL + "wiki/" + wiki.pageName␊
wikiPageSoup = BeautifulSoup(self.curl_get(wikiURL).getvalue(), "html.parser")␊
wikiContent = wikiPageSoup.find(id="wikicontent")␊
wiki.htmlContent = wikiContent.prettify()␊
wiki.textContent = wikiContent.get_text()␊
project.wikis.append(wiki)␊
␊
return project
services/srchub.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
from Service import Service␊
from Project import REPO_TYPES, Project␊
from Release import Release␊
from Issue import IssueComment, Issue␊
from Wiki import Wiki␊
from bs4 import BeautifulSoup␊
␊
class srchub(Service):␊
␊
DOMAIN = "https://beta.datanethost.net"␊
␊
def getProjects(self):␊
# Perhaps I should provide more API endpoints to make scraping easier...␊
projectlist = self.curl_get(self.DOMAIN + "/projects/").getvalue()␊
soup = BeautifulSoup(projectlist, "html.parser")␊
links = soup.find("ul", "prjlistclass")␊
projects = []␊
for link in links.find_all("a"):␊
project = Project()␊
sourceType = None␊
projectURL = self.DOMAIN + link.get("href")␊
projectName = projectURL.split("/")[-2]␊
␊
projectpageHTML = self.curl_get(projectURL).getvalue()␊
projectpageSoup = BeautifulSoup(projectpageHTML, "html.parser")␊
␊
sourceURL = projectpageSoup.find(name="a", string="Source").get("href")␊
sourceSoup = BeautifulSoup(self.curl_get(self.DOMAIN + sourceURL).getvalue(), "html.parser")␊
sourceSoupText = sourceSoup.get_text()␊
␊
# get source␊
if "git clone" in sourceSoupText:␊
project.repoType = REPO_TYPES.git␊
project.repoURL = "git://" + self.DOMAIN + "/" + projectName + ".git"␊
elif "svn co" in sourceSoupText:␊
project.repoType = REPO_TYPES.SVN␊
project.repoURL = "https://" + self.DOMAIN + "/svn/" + projectName + "/"␊
else:␊
project.repoType = REPO_TYPES.hg␊
project.repoURL = "https://" + self.DOMAIN + "/hg/" + projectName + "/"␊
␊
␊
# get downloads␊
project.releases = []␊
downlaodsSoup = BeautifulSoup(self.curl_get(projectURL + "downloads/").getvalue(), "html.parser")␊
downloadSection = downlaodsSoup.find("table", "uploads")␊
if "No downloads were found." not in downlaodsSoup.get_text():␊
downloadRows = downloadSection.find_all("tr")[1:]␊
for downloadRow in downloadRows:␊
cols = downloadRow.find_all("td")␊
downloadTD = cols[0]␊
downloadURL = self.DOMAIN + "/p/" + projectName + "/downloads/get/" + downloadTD.a.text␊
fileName = downloadTD.a.text␊
release = Release()␊
release.fileURL = downloadURL␊
release.fileName = fileName␊
project.releases.append(release)␊
␊
# get issues␊
project.issues = []␊
issuesSoup = BeautifulSoup(self.curl_get(projectURL + "issues/").getvalue(), "html.parser")␊
if "No issues were found." not in issuesSoup.get_text():␊
issuesSection = issuesSoup.find("table", "recent-issues")␊
for issueRow in issuesSection.find_all("tr")[1:]:␊
issue = Issue()␊
cols = issueRow.find_all("td")␊
issueId = cols[0].text␊
issueURL = projectURL + "issues/" + issueId + "/"␊
issueStatus = cols[2].text␊
issueSummary = cols[1].text␊
issueTitle = cols[1].find("a").text␊
issueAuthor = cols[3].text␊
issue.author = issueAuthor␊
issue.comments = []␊
issue.status = issueStatus␊
issue.summary = issueSummary␊
issue.title = issueTitle␊
issue.id = issueId␊
# we must go deeper to get comments␊
issueComments = BeautifulSoup(self.curl_get(issueURL).getvalue(), "html.parser")␊
for comment in issueComments.find_all("div", "issue-comment"):␊
author = comment.find("p").get_text().split("by")[1].split(",")[0]␊
date = comment.find("span").get_text()␊
commentText = comment.find("pre").get_text()␊
issueComment = IssueComment()␊
issueComment.date = date␊
issueComment.author = author␊
issueComment.summary = commentText␊
issue.comments.append(issueComment)␊
␊
project.issues.append(issue)␊
␊
# get wiki pages␊
project.wikis = []␊
wikiSoup = BeautifulSoup(self.curl_get(projectURL + "doc/").getvalue(), "html.parser")␊
if "No documentation pages were found." not in wikiSoup.get_text():␊
wikiSection = wikiSoup.find("table", "recent-issues")␊
for wikiRow in wikiSection.find_all("tr")[1:]:␊
wiki = Wiki()␊
cols = wikiRow.find_all("td")␊
wiki.pageName = cols[0].text␊
wiki.summary = cols[1].text␊
wiki.updated = cols[2].text␊
wikiURL = projectURL + "page/" + wiki.pageName + "/"␊
wikiPageSoup = BeautifulSoup(self.curl_get(wikiURL).getvalue(), "html.parser")␊
wikiContent = wikiPageSoup.find(id="wiki-content")␊
wiki.htmlContent = wikiContent.prettify()␊
wiki.textContent = wikiContent.get_text()␊
project.wikis.append(wiki)␊
␊
␊
projects.append(project)␊
␊
return projects␊
␊
␊
␊
␊
␊

Archive Download the corresponding diff file

Branches

Number of commits:
Page rendered in 0.08568s using 14 queries.