Twitter爬虫核心技术：全自动抓取世界上的热门主题以及转推,引用,回复的用户的最新微博.

import twitterdef oauth_login():# XXX: Go to /apps/new to create an app and get values # for these credentials that you'll need to provide in place of these# empty string values that are defined as placeholders.# See https:///docs/auth/oauth for more information# on Twitter's OAuth implementation.CONSUMER_KEY = ''CONSUMER_SECRET = ''OAUTH_TOKEN = ''OAUTH_TOKEN_SECRET = ''auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET,CONSUMER_KEY, CONSUMER_SECRET)twitter_api = twitter.Twitter(auth=auth)return twitter_api# Sample usagetwitter_api = oauth_login()# Nothing to see by displaying twitter_api except that it's now a# defined variableprint twitter_apiimport jsondef twitter_trends(twitter_api, woe_id):# Prefix ID with the underscore for query string parameterization.# Without the underscore, the twitter package appends the ID value# to the URL itself as a special-case keyword argument.return twitter_api.trends.place(_id=woe_id)def twitter_search(twitter_api, q, max_results=2000000, **kw):# See https:///docs/api/1.1/get/search/tweets and# https:///docs/using-search for details on advanced# search criteria that may be useful for keyword arguments# See https:///docs/api/1.1/get/search/tweetssearch_results = twitter_api.search.tweets(q=q, count=200000, **kw)statuses = search_results['statuses']# Iterate through batches of results by following the cursor until we# reach the desired number of results, keeping in mind that OAuth users # can "only" make 180 search queries per 15-minute interval. See# https:///docs/rate-limiting/1.1/limits# for details. A reasonable number of results is ~1000, although# that number of results may not exist for all queries.# Enforce a reasonable limitmax_results = min(1000, max_results)for _ in range(10): # 10*100 = 1000try:next_results = search_results['search_metadata']['next_results'] except KeyError, e: # No more results when next_results doesn't exist break# Create a dictionary from next_results, which has the following form:# ?max_id=313519052523986943&q=NCAA&include_entities=1kwargs = dict([ kv.split('=')for kv in next_results[1:].split("&") ])search_results = twitter_api.search.tweets(**kwargs)statuses += search_results['statuses']if len(statuses) > max_results:breakreturn statusesdef extract_tweet_entities(statuses):# See https:///docs/tweet-entities for more details on tweet# entitiesif len(statuses) == 0:return [], [], [], [], []screen_names = [ user_mention['screen_name']for status in statusesfor user_mention in status['entities']['user_mentions'] ]hashtags = [ hashtag['text']for status in statusesfor hashtag in status['entities']['hashtags'] ]urls = [ url['expanded_url']for status in statusesfor url in status['entities']['urls'] ]symbols = [ symbol['text']for status in statusesfor symbol in status['entities']['symbols'] ]# In some circumstances (such as search results), the media entity# may not appearif status['entities'].has_key('media'):media = [ media['url']for status in statusesfor media in status['entities']['media'] ] else:media = []return screen_names, hashtags, urls, media, symbolsdef find_popular_tweets(twitter_api, statuses, retweet_threshold=3):# You could also consider using the favorite_count parameter as part of# this heuristic, possibly using it to provide an additional boost to# popular tweets in a ranked formulationreturn [ statusfor status in statusesif status['retweet_count'] > retweet_threshold ]from collections import Counterdef get_common_tweet_entities(statuses, entity_threshold=3):# Create a flat list of all tweet entitiestweet_entities = [ efor status in statusesfor entity_type in extract_tweet_entities([status])for e in entity_type]c = Counter(tweet_entities).most_common()# Compute frequenciesreturn [ (k,v)for (k,v) in cif v >= entity_threshold]import redef get_rt_attributions(tweet):# Regex adapted from Stack Overflow (http://bit.ly/1821y0J)rt_patterns = pile(r"(RT|via)((?:\b\W*@\w+)+)", re.IGNORECASE)rt_attributions = []# Inspect the tweet to see if it was produced with /statuses/retweet/:id.# See https:///docs/api/1.1/get/statuses/retweets/%3Aid.if tweet.has_key('retweeted_status'):attribution = tweet['retweeted_status']['user']['screen_name'].lower()rt_attributions.append(attribution)# Also, inspect the tweet for the presence of "legacy" retweet patterns# such as "RT" and "via", which are still widely used for various reasons# and potentially very useful. See https:///discussions/2847# and https:///discussions/1748 for some details on how/why.try:rt_attributions += [mention.strip()for mention in rt_patterns.findall(tweet['text'])[0][1].split()]except IndexError, e:pass# Filter out any duplicatesreturn list(set([rta.strip("@").lower() for rta in rt_attributions]))import pymongo # pip install pymongodef save_to_mongo(data, mongo_db, mongo_db_coll, **mongo_conn_kw):# Connects to the MongoDB server running on# localhost:27017 by defaultclient = pymongo.MongoClient(**mongo_conn_kw)# Get a reference to a particular databasedb = client[mongo_db]# Reference a particular collection in the databasecoll = db[mongo_db_coll]# Perform a bulk insert and return the IDsreturn coll.insert(data)def load_from_mongo(mongo_db, mongo_db_coll, return_cursor=False,criteria=None, projection=None, **mongo_conn_kw):# Optionally, use criteria and projection to limit the data that is# returned as documented in# /manual/reference/method/db.collection.find/# Consider leveraging MongoDB's aggregations framework for more# sophisticated queries.client = pymongo.MongoClient(**mongo_conn_kw)db = client[mongo_db]coll = db[mongo_db_coll]if criteria is None:criteria = {}if projection is None:cursor = coll.find(criteria)else:cursor = coll.find(criteria, projection)# Returning a cursor is recommended for large amounts of dataif return_cursor:return cursorelse:return [ item for item in cursor ]#save_to_mongo(results, 'search_results005', q)##load_from_mongo('search_results005', q)import sysimport timefrom urllib2 import URLErrorfrom httplib import BadStatusLinedef make_twitter_request(twitter_api_func, max_errors=10, *args, **kw):# A nested helper function that handles common HTTPErrors. Return an updated # value for wait_period if the problem is a 500 level error. Block until the# rate limit is reset if it's a rate limiting issue (429 error). Returns None# for 401 and 404 errors, which requires special handling by the caller.def handle_twitter_http_error(e, wait_period=2, sleep_when_rate_limited=True):if wait_period > 3600: # Secondsprint >> sys.stderr, 'Too many retries. Quitting.'raise e# See https:///docs/error-codes-responses for common codesif e.e.code == 401:print >> sys.stderr, 'Encountered 401 Error (Not Authorized)'return Noneelif e.e.code == 404:print >> sys.stderr, 'Encountered 404 Error (Not Found)'return Noneelif e.e.code == 429:print >> sys.stderr, 'Encountered 429 Error (Rate Limit Exceeded)'if sleep_when_rate_limited:print >> sys.stderr, "Retrying in 15 minutes...ZzZ..."sys.stderr.flush()time.sleep(60*15 + 5)print >> sys.stderr, '...ZzZ...Awake now and trying again.'return 2else:raise e # Caller must handle the rate limiting issueelif e.e.code in (500, 502, 503, 504):print >> sys.stderr, 'Encountered %i Error. Retrying in %i seconds' % \(e.e.code, wait_period)time.sleep(wait_period)wait_period *= 1.5return wait_periodelse:raise e# End of nested helper functionwait_period = 2error_count = 0while True:try:return twitter_api_func(*args, **kw)except twitter.api.TwitterHTTPError, e:error_count = 0wait_period = handle_twitter_http_error(e, wait_period)if wait_period is None:returnexcept URLError, e:error_count += 1print >> sys.stderr, "URLError encountered. Continuing."if error_count > max_errors:print >> sys.stderr, "Too many consecutive errors...bailing out."raiseexcept BadStatusLine, e:error_count += 1print >> sys.stderr, "BadStatusLine encountered. Continuing."if error_count > max_errors:print >> sys.stderr, "Too many consecutive errors...bailing out."raisedef harvest_user_timeline(twitter_api, screen_name=None, user_id=None, max_results=1000):assert (screen_name != None) != (user_id != None), \"Must have screen_name or user_id, but not both"kw = { # Keyword args for the Twitter API call'count': 200,'trim_user': 'true','include_rts' : 'true','since_id' : 1}if screen_name:kw['screen_name'] = screen_nameelse:kw['user_id'] = user_idmax_pages = 10results = []tweets = make_twitter_request(twitter_er_timeline, **kw)if tweets is None: # 401 (Not Authorized) - Need to bail out on loop entrytweets = []results += tweetsprint >> sys.stderr, 'Fetched %i tweets' % len(tweets)page_num = 1# Many Twitter accounts have fewer than 200 tweets so you don't want to enter# the loop and waste a precious request if max_results = 200.# Note: Analogous optimizations could be applied inside the loop to try and# save requests. e.g. Don't make a third request if you have 287 tweets out of# a possible 400 tweets after your second request. Twitter does do some# post-filtering on censored and deleted tweets out of batches of 'count', though,# so you can't strictly check for the number of results being 200. You might get# back 198, for example, and still have many more tweets to go. If you have the# total number of tweets for an account (by GET /users/lookup/), then you could# simply use this value as a guide.if max_results == kw['count']:page_num = max_pages # Prevent loop entrywhile page_num < max_pages and len(tweets) > 0 and len(results) < max_results:# Necessary for traversing the timeline in Twitter's v1.1 API:# get the next query's max-id parameter to pass in.# See https:///docs/working-with-timelines.kw['max_id'] = min([ tweet['id'] for tweet in tweets]) - 1tweets = make_twitter_request(twitter_er_timeline, **kw)results += tweetsprint >> sys.stderr, 'Fetched %i tweets' % (len(tweets),)page_num += 1print >> sys.stderr, 'Done fetching tweets'return results[:max_results]WORLD_WOE_ID = 1world_trends = twitter_trends(twitter_api, WORLD_WOE_ID)#print json.dumps(world_trends, indent=1)for i in range(10):q=world_trends[0]['trends'][i]['name']print "第",i,"个流行的主题是：", q# See https:///docs/api/1.1/get/search/tweetssearch_results = twitter_api.search.tweets(q=q, count=200000)statuses = search_results['statuses']save_to_mongo(statuses, 'data201609061','search-results')# Iterate through 5 more batches of results by following theprint "第一次爬取", len(statuses),"条tweets"t=len(statuses)for i in range(t):tweets = harvest_user_timeline(twitter_api, screen_name=statuses[i]['user']['screen_name'], \max_results=200)if tweets==[]: # 401 (Not Authorized) - Need to bail out on loop entrycontinue#Save to MongoDB with save_to_mongo or a local file with save_json...#popular_tweets = find_popular_tweets(twitter_api, tweets)#save_to_mongo(tweets, 'data20160827','followers_tweets')save_to_mongo(tweets, 'data201609061','search-results')if 'retweeted_status' in statuses[i]:tweets2 = harvest_user_timeline(twitter_api, screen_name=statuses[i]['retweeted_status']['user']['screen_name'], \max_results=200)if tweets2==[]: # 401 (Not Authorized) - Need to bail out on loop entrycontinuesave_to_mongo(tweets2, 'data201609061','search-results')print "有转推哈哈"if 'quoted_status' in statuses[i]:tweets3 = harvest_user_timeline(twitter_api, screen_name=statuses[i]['quoted_status']['user']['screen_name'], \max_results=200)if tweets3==[]: # 401 (Not Authorized) - Need to bail out on loop entrycontinuesave_to_mongo(tweets3, 'data201609061','search-results')print "有引用哈哈"if statuses[i]['in_reply_to_screen_name']!=None:tweets4 = harvest_user_timeline(twitter_api, screen_name=statuses[i]['in_reply_to_screen_name'], \max_results=200)if tweets4==[]: # 401 (Not Authorized) - Need to bail out on loop entrycontinuesave_to_mongo(tweets4, 'data201609061','search-results')print "有回复哈哈"for _ in range(5000):try:next_results = search_results['search_metadata']['next_results']except KeyError, e: # No more results when next_results doesn't existbreak# Create a dictionary from next_results, which has the following form:# ?max_id=313519052523986943&q=NCAA&include_entities=1kwargs = dict([ kv.split('=') for kv in next_results[1:].split("&") ])results=[]search_results = twitter_api.search.tweets(**kwargs)results= search_results['statuses']save_to_mongo(results, 'data201609061','search-results')print "第二次爬取", len(results),"条tweets"tt=len(results)for ii in range(tt):tweets5 = harvest_user_timeline(twitter_api, screen_name=results[ii]['user']['screen_name'], \max_results=200)if tweets5==[]: # 401 (Not Authorized) - Need to bail out on loop entrycontinue#Save to MongoDB with save_to_mongo or a local file with save_json...#popular_tweets = find_popular_tweets(twitter_api, tweets)#save_to_mongo(tweets, 'data20160827','followers_tweets')save_to_mongo(tweets5, 'data201609061','search-results')if 'retweeted_status' in results[ii]:tweets6 = harvest_user_timeline(twitter_api, screen_name=results[ii]['retweeted_status']['user']['screen_name'], \max_results=200)if tweets6==[]: # 401 (Not Authorized) - Need to bail out on loop entrycontinuesave_to_mongo(tweets6, 'data201609061','search-results')print "还是有转推哈哈"if 'quoted_status' in results[ii]:tweets7 = harvest_user_timeline(twitter_api, screen_name=results[ii]['quoted_status']['user']['screen_name'], \max_results=200)if tweets7==[]: # 401 (Not Authorized) - Need to bail out on loop entrycontinuesave_to_mongo(tweets7, 'data201609061','search-results')print "还是有引用哈哈"if results[ii]['in_reply_to_screen_name']!=None:tweets8 = harvest_user_timeline(twitter_api, screen_name=results[ii]['in_reply_to_screen_name'], \max_results=200)if tweets8==[]: # 401 (Not Authorized) - Need to bail out on loop entrycontinuesave_to_mongo(tweets8, 'data201609061','search-results')print "还是有回复哈哈"。