从指定 offset 消费 Kafka

pip install kafka-python

import gzip
from kafka import KafkaConsumer
from kafka import TopicPartition

consumer = KafkaConsumer(bootstrap_servers='10.30.10.15:9092')
partition = TopicPartition('dc-diagnostic-report', 0)
start = 8833
end = 8835
consumer.assign([partition])
consumer.seek(partition, start)

i=start
for msg in consumer:
    if msg.offset > end:
        break
    else:
        print msg
        try:
            outF = file( '/data/app/tar/' + str(i) + ".gz", 'wb')
            outF.write(msg.value)
            outF.close()
            i+=1
            print i
        except Exception,e:
            print e