2015-12-22 12:52:22 +08:00
|
|
|
language: ruby
|
|
|
|
|
rvm:
|
|
|
|
|
- 2.2
|
|
|
|
|
before_script:
|
|
|
|
|
- gem install awesome_bot
|
2015-10-13 10:20:43 +08:00
|
|
|
script:
|
2016-01-25 23:49:50 +08:00
|
|
|
- site404=www.datawrangling.com,getglue-data.s3.amazonaws.com,archive.org/details/2011-05-calufa-twitter-sql,www.stats4stem.org,lib.stat.cmu.edu,http://www.oecd.org/document/0,data.gov.be,census.gov/acs/www/data_documentation/data_release_info/,europeansocialsurvey.org/data/
|
2016-01-31 21:49:50 +08:00
|
|
|
- whtlist=travis,crawdad.cs.dartmouth.edu,data.nasdaq.com,137.189.35.203/WebUI/CatDatabase/catData.html,numbrary.com,www.cmr.osu.edu,wiki.earthdata.nasa.gov,missionlocal.org,gutenberg.org,donnees.gouv.qc.ca,data.rio.rj.gov.br,cvcl.mit.edu,data.ohouston.org,ntrl.ntis.gov,networkdata.ics.uci.edu,sinda.crn2.inpe.br,archive.ics.uci.edu,openflights.org
|
|
|
|
|
- site503=labrosa.ee.columbia.edu/millionsong,datamob.org,research.microsoft.com
|
2016-01-31 21:53:53 +08:00
|
|
|
- awesome_bot README.rst --allow-dupe --allow-redirect --white-list $site404,$whtlist,$site503 --set-timeout=5
|
