python 高度健壮性爬虫的异常和超时问题

  try :      passhttp : / / top . jobbole . com / deliver - article / #      #可能出错的语句 except Exception , e :      pass      #保留错误的url，留待下次重跑      print e finally :      #无论是否处理了异常都继续运行      print time . ctime ( )

import requests requests . get ( url , timeout = 60 )

import requesocks session = requesocks . session ( ) response = session . get ( URL , headers = headers , timeout = 10 )

from selenium import webdriver from selenium . webdriver . common . by import By from selenium . webdriver . support . ui import WebDriverWait from selenium . webdriver . support import expected_conditions as EC   driver = webdriver . Firefox ( ) driver . get ( "http://somedomain/url_that_delays_loading" ) try :      element = WebDriverWait ( driver , 10 ) . until (   #这里修改时间          EC . presence_of_element_located ( ( By . ID , "myDynamicElement" ) )      ) finally :      driver . quit ( )

from selenium import webdriver   driver = webdriver . Firefox ( ) driver . implicitly_wait ( 10 ) # seconds driver . get ( "http://somedomain/url_that_delays_loading" ) myDynamicElement = driver . find_element_by_id ( "myDynamicElement" )

page . settings . resourceTimeout = 5000 ; // 等待5秒   var system = require ( 'system' ) ; var args = system . args ; var url = args [ 1 ] ; var page = require ( 'webpage' ) . create ( ) ; page . settings . resourceTimeout = 5000 ; // 等待5秒 page . onResourceTimeout = function ( e ) { console . log ( e . errorCode ) ; & nbsp ; & nbsp ; //打印错误码 console . log ( e . errorString ) ; //打印错误语句 console . log ( e . url ) ; & nbsp ; & nbsp ; & nbsp ; & nbsp ; //打印错误url phantom . exit ( 1 ) ; } ; page . open ( url , function ( status ) { if ( status === 'success' ) { var html = page . evaluate ( function ( ) { returndocument . documentElement . outerHTML ; } ) ; console . log ( html ) ; } phantom . exit ( ) ; } ) ; //$phantomjs xx.js http://bbs.pcbaby.com.cn/topic-2149414.html

#coding:utf-8 import time import signal   def test ( i ) :      time . sleep ( 0.999 ) #模拟超时的情况      print "%d within time" % ( i )      return i   def fuc_time ( time_out ) :      # 此为函数超时控制，替换下面的test函数为可能出现未知错误死锁的函数      def handler ( signum , frame ) :          raise AssertionError      try :          signal . signal ( signal . SIGALRM , handler )          signal . alarm ( time_out ) #time_out为超时时间          temp = test ( 1 ) #函数设置部分，如果未超时则正常返回数据，          return temp      except AssertionError :          print "%d timeout" % ( i ) # 超时则报错   if __name__ == '__main__' :      for i in range ( 1 , 10 ) :          fuc_time ( 1 )

import subprocess from threading import Timer import time   kill = lambda process : process . kill ( )   cmd = [ "ping" , "www.google.com" ] ping = subprocess . Popen (      cmd , stdout = subprocess . PIPE , stderr = subprocess . PIPE )   my_timer = Timer ( 5 , kill , [ ping ] ) #这里设定时间，和命令 try :      my_timer . start ( ) #启用      stdout , stderr = ping . communicate ( ) #获得输出      #print stderr      print time . ctime ( ) finally :      print time . ctime ( )      my_timer . cancel ( )

import time import sys import os def restart_program ( ) :    python = sys . executable    os . execl ( python , python , * sys . argv )    if __name__ == "__main__" :    print 'start...'    print u "3秒后,程序将结束..." . encode ( "utf8" )    time . sleep ( 3 )    restart_program ( )