python之多线程与多进程

关键词： GIL锁，IO繁忙，线程安全，线程同步，进程池，进程通信，队列

GIL锁;

全局解释锁，Global Interpretor Lock,
作用:单位时间内只允许一个线程运行。
结果：无法利用多核实现并发功能，cpu繁忙的任务无效甚至效率更低（进程切换需要耗费资源），因此在python中不建议使用多线程，而使用多进程代替
cpu繁忙与io繁忙

cpu繁忙：内存操作多

io繁忙：硬盘或其他存储介质读写多，比如各种数据中心，网络存储和云存服务器
线程同步：

单位时间只能有一个线程访问某个资源
线程安全：

使用了线程同步，保证资源正确调用

python实现多进程的方法：

主角：multiprocessing
工具：Process（进程转化为类）, Pool（进程池）, Queue（队列，用于进程通讯）

初级实现：

class Myprocess(Process): #自定义线程类

    def __init__(self, wait_ime):
        Process.__init__(self)
        self.wait_ime = wait_ime

    def run(self):#重写run方法！！！！！！重要
        n = 0
        while n < 4:
            print ("subProcess %s run," % os.getpid(), "{0}".format(time.ctime()))
            time.sleep(self.wait_ime)
            n += 1
if __name__ == "__main__":

    p=Myprocess(2)
    q=Myprocess(0.5)
    p.start() #启动进程
    q.start()
    p.join()  #等待p进程结束，主进程才能往下运行（简称阻塞）
    q.join()
    print('结束')

结果

subProcess 6600 run, Thu Nov 22 16:19:57 2018  # 两个同进程
subProcess 3012 run, Thu Nov 22 16:19:57 2018
subProcess 3012 run, Thu Nov 22 16:19:58 2018
subProcess 3012 run, Thu Nov 22 16:19:58 2018
subProcess 3012 run, Thu Nov 22 16:19:59 2018
subProcess 6600 run, Thu Nov 22 16:19:59 2018
subProcess 6600 run, Thu Nov 22 16:20:01 2018
subProcess 6600 run, Thu Nov 22 16:20:03 2018
结束

进程池

def example(wait_ime):
        n = 0
        while n < 4:
            print ("subProcess %s run," % os.getpid(), "{0}".format(time.ctime()))
            time.sleep(wait_ime)
            n += 1
if __name__ == "__main__":
    pool=Pool(5)

    for i in (0.1,1.2,0.5):
        pool.apply_async(example,(i,))#非阻塞模式，不会等一个进程运行完才运行下一个进程
    pool.close()
    pool.join()
    print('结束')

队列（实现生产者与消费者模式的核心工具）

 class MultiProcessProducer(multiprocessing.Process):#生产者类
    def __init__(self, num, queue):              # 传出队列queue
       """Constructor"""
       multiprocessing.Process.__init__(self)
       self.num = num
       self.queue = queue
 
    def run(self):                                   #存入数字
       t1 = time.time()
       print('producer start ' + str(self.num))
       for i in range(1000):
          self.queue.put((i, self.num))          #插入队列
          time.sleep(0.1)
          print ('producer put', i, self.num)
       t2 = time.time()
 
       print('producer exit ' + str(self.num))
       use_time = str(t2 - t1)
       print('producer ' + str(self.num) + ', \
       use_time: '+ use_time)
 
 
 
 class MultiProcessConsumer(multiprocessing.Process):# 消费者类
     def __init__(self, num, queue):
       """Constructor"""
       multiprocessing.Process.__init__(self)
       self.num = num
       self.queue = queue                            #传入队列
 
     def run(self):
         t1 = time.time()
         print('consumer start ' + str(self.num))
         while True:
             d = self.queue.get()
             print('get'+str(d))
         t2 = time.time()
         print('consumer exit ' + str(self.num))
         print('consumer ' + str(self.num) + ', use time:'+ str(t2 - t1))
 
 def main():
        # create queue
        queue = multiprocessing.Queue()
 
        # create processes
        producer = []
        for i in range(5):
           producer.append(MultiProcessProducer(i, queue))
 
        consumer = []
        for i in range(5):
           consumer.append(MultiProcessConsumer(i, queue))
 
        # start processes
        for i in range(len(producer)):
           producer[i].start()
 
        for i in range(len(consumer)):
           consumer[i].start()
 
        # wait for processs to exit
        for i in range(len(producer)):
           producer[i].join()
 
        for i in range(len(consumer)):
           queue.put(None)
 
        for i in range(len(consumer)):
           consumer[i].join()
 
        print('all done finish')