python实现RabbitMQ同步跟异步消费模型

python实现RabbitMQ同步跟异步消费模型
1，消息推送类
1 import pika
2
3
4# 同步消息推送类
5 class RabbitPublisher(object):
6
7# 传⼊RabbitMQ的ip，⽤户名，密码，实例化⼀个管道
8 def __init__(self, host, user, password):
9 self.host = host
10 er = user
11 self.password = password
12 self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(er, self.password)))
13 self.channel = self.connection.channel()
14
15# 发送消息在队列中
16 def send(self, queue_name, body):
17 self.channel.queue_declare(queue=queue_name, durable=True) # 声明⼀个持久化队列
18 self.channel.basic_publish(exchange='',
19 routing_key=queue_name, # 队列名字
20 body=body, # 消息内容
21 properties=pika.BasicProperties(
22 delivery_mode=2, # 消息持久化
23 ))
24
25# 清除指定队列的所有的消息
26 def purge(self, queue_name):
27 self.channel.queue_purge(queue_name)
28
29# 删除指定队列
30 def delete(self, queue_name, if_unused=False, if_empty=False):
31 self.channel.queue_delete(queue_name, if_unused=if_unused, if_empty=if_empty)
32
33# 断开连接
34 def stop(self):
35 self.connection.close()
View Code
2.消息消费类
（1）同步消息消费
在同步消息消费的时候可能会出现pika库断开的情况，原因是因为pika客户端没有及时发送⼼跳，连接就被server端断开了。

解决⽅案就是做⼀个⼼跳线程来维护连接。

⼼跳线程类
1 class Heartbeat(threading.Thread):
2
3 def __init__(self, connection):
4 super(Heartbeat, self).__init__()
5 self.lock = threading.Lock() # 线程锁
6 self.connection = connection # rabbit连接
7 self.quitflag = False# 退出标志
8 self.stopflag = True# 暂停标志
9 self.setDaemon(True) # 设置为守护线程，当消息处理完，⾃动清除
10
11# 间隔10s发送⼼跳
12 def run(self):
13while not self.quitflag:
14 time.sleep(10) # 睡10s发⼀次⼼跳
15 self.lock.acquire() # 加线程锁
16if self.stopflag:
17 self.lock.release()
18continue
19 try:
20 self.connection.process_data_events() # ⼀直等待服务段发来的消息
21 except Exception as e:
22 print "Error format: %s" % (str(e))
23 self.lock.release()
24return
25 self.lock.release()
26
27# 开启⼼跳保护
28 def startheartbeat(self):
29 self.lock.acquire()
30if self.quitflag:
31 self.lock.release()
32return
33 self.stopflag = False
34 self.lock.release()
View Code
消息消费类
1# 同步消息消费类
2 class RabbitConsumer(object):
3
4# 传⼊RabbitMQ的ip，⽤户名，密码，实例化⼀个管道
5 def __init__(self, host, user, password):
6 self.host = host
7 er = user
8 self.password = password
9 self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(er, self.password)))
10 self.channel = self.connection.channel()
11
12# 进⾏消费
13 def receive(self, queue_name, callback_worker, prefetch_count=1): # callback_worker为消费的回调函数
14 self.channel.queue_declare(queue=queue_name, durable=True)
15 self.channel.basic_qos(prefetch_count=prefetch_count) # 设置预取的数量，如果为0则不预取，消费者处理越快，可以将这个这设置的越⾼
16 self.channel.basic_consume(callback_worker, queue=queue_name) # callback_worker为消费的回调函数
17 heartbeat = Heartbeat(self.connection) # 实例化⼀个⼼跳类
18 heartbeat.start() # 开启⼀个⼼跳线程，不传target的值默认运⾏run函数
19 heartbeat.startheartbeat() # 开启⼼跳保护
20 self.channel.start_consuming() # 开始消费
View Code
调⽤⽅法
# 消费回调函数
def callback(ch, method, properties, body):
print(" [x] Received %r" % body)
ch.basic_ack(delivery_tag=method.delivery_tag) # 告诉⽣产者处理完成
consumer = RabbitConsumer(host="12.12.12.12", user="test", password="123")
consumer.receive(queue_name="queue1", callback_worker=callback)
（2）异步消息消费（推荐）
pika提供了⽀持异步发送模式的selectconnection⽅法⽀持异步发送接收（通过回调的⽅式）
在连接的时候stop_ioloop_on_close=False需要低版本的pika，⽐如0.13.1，安装⽅式 pip install pika==0.13.1
connectioon建⽴时回调建⽴channel， channel建⽴时⼀次回调各种declare⽅法，declare建⽴时依次回调publish。

同使⽤blockconnection⽅法相⽐，通过wireshark抓包来看，使⽤异步的⽅式会对发包进⾏⼀些优化，会将⼏个包合并成⼀个⼤包，然后做⼀次ack应答从⽽提⾼效率，与之相反使⽤blockconnection时将会做⾄少两次ack，head⼀次content⼀次等
因此再试⽤异步的⽅式时会获得⼀定的优化
异步消息消费类
1# 异步消息消费类
2 class RabbitConsumerAsync(object):
3 EXCHANGE = 'amq.direct'
4 EXCHANGE_TYPE = 'direct'
5
6 def __init__(self, host, user, password, queue_name="fish_test", callback_worker=None, prefetch_count=1):
7 self.host = host
8 er = user
9 self.password = password
10 self._connection = None
11 self._channel = None
12 self._closing = False
13 self._consumer_tag = None
14 self.QUEUE = queue_name
15 self.callbackworker = callback_worker
16 self.prefetch_count = prefetch_count
17
18 def connect(self):
19return pika.SelectConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(er, self.password)), self.on_connection_open, 20 stop_ioloop_on_close=False)
21
22 def on_connection_open(self, unused_connection):
23 self.add_on_connection_close_callback()
24 self.open_channel()
25
26 def add_on_connection_close_callback(self):
27 self._connection.add_on_close_callback(self.on_connection_closed)
28
29 def on_connection_closed(self, connection, reply_code, reply_text):
30 self._channel = None
31if self._closing:
32 self._connection.ioloop.stop()
33else:
34 self._connection.add_timeout(5, self.reconnect)
35
36 def reconnect(self):
37 self._connection.ioloop.stop()
38if not self._closing:
39 self._connection = self.connect()
40 self._connection.ioloop.start()
41
42 def open_channel(self):
43 self._connection.channel(on_open_callback=self.on_channel_open)
44
45 def on_channel_open(self, channel):
46 self._channel = channel
47 self._channel.basic_qos(prefetch_count=self.prefetch_count)
48 self.add_on_channel_close_callback()
49 self.setup_exchange(self.EXCHANGE)
50
51 def add_on_channel_close_callback(self):
52 self._channel.add_on_close_callback(self.on_channel_closed)
53
54 def on_channel_closed(self, channel, reply_code, reply_text):
55 print reply_text
56 self._connection.close()
57
58 def setup_exchange(self, exchange_name):
59 self._channel.exchange_declare(self.on_exchange_declareok, exchange_name, self.EXCHANGE_TYPE, durable=True) 60
61 def on_exchange_declareok(self, unused_frame):
62 self.setup_queue()
63
64 def setup_queue(self):
65 self._channel.queue_declare(self.on_queue_declareok, self.QUEUE, durable=True)
66
67 def on_queue_declareok(self, method_frame):
68 self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE, self.QUEUE)
69
70 def on_bindok(self, unused_frame):
71 self.start_consuming()
72
73 def start_consuming(self):
74 self.add_on_cancel_callback()
75 self._consumer_tag = self._channel.basic_consume(self.on_message, self.QUEUE)
76
77 def add_on_cancel_callback(self):
78 self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
79
80 def on_consumer_cancelled(self, method_frame):
81if self._channel:
82 self._channel.close()
83
84 def on_message(self, unused_channel, basic_deliver, properties, body):
85 self.callbackworker(body)
86 self.acknowledge_message(basic_deliver.delivery_tag)
87
88 def acknowledge_message(self, delivery_tag):
89 self._channel.basic_ack(delivery_tag)
90
91 def stop_consuming(self):
92if self._channel:
93 self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
94
95 def on_cancelok(self, unused_frame):
96 self.close_channel()
97
98 def close_channel(self):
99 self._channel.close()
100
101 def run(self):
102 self._connection = self.connect()
103 self._connection.ioloop.start()
104
105 def stop(self):
106 self._closing = True
107 self.stop_consuming()
108 self._connection.ioloop.start()
109
110 def close_connection(self):
111 self._connection.close()
View Code
调⽤⽅法
# 消费回调函数
def callback(body):
print(" [x] Received %r" % body)
consumer = RabbitConsumerAsync(host="12.12.12.12", user="test", password="123", queue_name="fish_test", callback_worker=callback, prefetch_count=2) consumer.run()
（后⾯这两个可不加⼊）守护进程类（保证消费运⾏）
class CDaemon(object):
"""
a generic daemon class.
usage: subclass the CDaemon class and override the run() method
stderr 表⽰错误⽇志⽂件绝对路径, 收集启动过程中的错误⽇志
verbose 表⽰将启动运⾏过程中的异常错误信息打印到终端,便于调试,建议⾮调试模式下关闭, 默认为1, 表⽰开启
save_path 表⽰守护进程pid⽂件的绝对路径
"""
def__init__(self, save_path, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1):
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.pidfile = save_path # pid⽂件绝对路径
self.home_dir = home_dir
self.verbose = verbose # 调试开关
self.umask = umask
self.daemon_alive = True
def daemonize(self):
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError, e:
sys.stderr.write('fork #1 failed: %d (%s)\n' % (e.errno, e.strerror))
sys.exit(1)
os.chdir(self.home_dir)
os.setsid()
os.umask(self.umask)
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError, e:
sys.stderr.write('fork #2 failed: %d (%s)\n' % (e.errno, e.strerror))
sys.exit(1)
sys.stdout.flush()
sys.stderr.flush()
si = file(self.stdin, 'r')
so = file(self.stdout, 'a+')
if self.stderr:
se = file(self.stderr, 'a+', 0)
else:
se = so
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
def sig_handler(signum, frame):
self.daemon_alive = False
signal.signal(signal.SIGTERM, sig_handler)
signal.signal(signal.SIGINT, sig_handler)
if self.verbose >= 1:
print'daemon process started ...'
atexit.register(self.del_pid)
pid = str(os.getpid())
file(self.pidfile, 'w+').write('%s\n' % pid)
def get_pid(self):
try:
pf = file(self.pidfile, 'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
except SystemExit:
pid = None
return pid
def del_pid(self):
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
def start(self, *args, **kwargs):
if self.verbose >= 1:
print'ready to starting ......'
# check for a pid file to see if the daemon already runs
pid = self.get_pid()
if pid:
msg = 'pid file %s already exists, is it already running?\n'
sys.stderr.write(msg % self.pidfile)
sys.exit(0)
# start the daemon
self.daemonize()
self.run(*args, **kwargs)
def stop(self):
if self.verbose >= 1:
print'stopping ...'
pid = self.get_pid()
if not pid:
msg = 'pid file [%s] does not exist. Not running?\n' % self.pidfile
sys.stderr.write(msg)
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
return
# try to kill the daemon process
try:
i = 0
while 1:
os.kill(pid, signal.SIGTERM)
time.sleep(0.1)
i = i + 1
if i % 10 == 0:
os.kill(pid, signal.SIGHUP)
except OSError, err:
err = str(err)
if err.find('No such process') > 0:
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
else:
print str(err)
sys.exit(1)
if self.verbose >= 1:
print'Stopped!'
def restart(self, *args, **kwargs):
self.stop()
self.start(*args, **kwargs)
def is_running(self):
pid = self.get_pid()
# print(pid)
return pid and os.path.exists('/proc/%d' % pid)
def run(self, *args, **kwargs):
# NOTE: override the method in subclass
print'base class run()'
View Code
调⽤
class RabbitDaemon(CDaemon):
def__init__(self, name, save_path, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1): CDaemon.__init__(self, save_path, stdin, stdout, stderr, home_dir, umask, verbose)
= name # 派⽣守护进程类的名称
def run(self, **kwargs):
# 新建⼀个队列链接
rab_con = RabbitConsumerAysnc(queuename="test", callbackworker=liando_sf_consumer, prefetch_count=2)
# 开启消费者进程
rab_con.run()
p_name = 'test'# 守护进程名称
pid_fn = '/www/rabbit/test.pid'# 守护进程pid⽂件的绝对路径
err_fn = '/www/rabbit/test_err.log'# 守护进程启动过程中的错误⽇志,内部出错能从这⾥看到cD = RabbitDaemon(p_name, pid_fn, stderr=err_fn, verbose=1)
View Code
参考链接。