前瞻

1. 上一篇文章写了文章python爬去天气的简单实用，但是你用的时间长了。
2. 你就会中国天气网返回一直是一个不变的数据,这当然不行，简直是恶心到爆炸,这个时候就会
需要自己去一个精确的接口
3. 除了中国的天气网,突然发现有一个中国天气台的网站,同构chrome游览器自带的工具

中国天气台的数据

网站分析

中国气象台

我们会发现很明显的有一个城市ID的存在,但是我们并不能找到所有城市的ID，这就很难受了。

当然，遍历所有的的数据也可以找到,第一遍我的思路是从1开始，但是发现很长一段时间都没有数据出现，这个时候我想到了我们应该以北京开始，查到北京的ID是54511,这个时候我们就可以从54511开始遍历，一直到59981，别问我为什，因为最后中国大型城市不可能超过1万。

我们把拿到的数据写入xls中,在分析相关东西

#! /usr/bin/env python
# -*- coding: utf-8 -*-

import urllib2
import json
import xlwt


def get_json_info(url):
    # 模拟浏览器访问
    header = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
              "Accept-Encoding": "gbk,utf-8,gb2312",
              "Accept-Language": "zh-CN,zh;q=0.8",
              "User-Agent": "Mozilla/5.0(Windows NT 10.0; WOW64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
              "Connection": "keep-alive"}
    request = urllib2.Request(url, headers=header)
    response = urllib2.urlopen(request)
    text = response.read()
    try:
        weatherJSON = json.JSONDecoder().decode(text)
    except:
        weatherJSON = {}

    print weatherJSON
    province = None
    city = None
    code = None
    city_url = None
    if weatherJSON.has_key('station'):
        station = weatherJSON['station']
        city = station.get('city', '无')
        province = station.get('province', '无')
        code = station.get('code', '无')
        city_url = station.get('city_url', '无')
        return province, city, code, city_url
    else:
        return province, city, code, city_url


def write_xls(city_list, province_list, code_list, city_url_list):
    file_name = './weather_data2.xls'
    wbk = xlwt.Workbook(encoding='utf-8', style_compression=0)
    sheet = wbk.add_sheet('weather_data', cell_overwrite_ok=True)

    alignment = xlwt.Alignment()  # Create Alignment
    alignment.horz = xlwt.Alignment.HORZ_CENTER  # 水平居中
    alignment.vert = xlwt.Alignment.VERT_CENTER  # 垂直居中
    style = xlwt.XFStyle()  # Create Style
    style.alignment = alignment  # Add Alignment to Style

    sheet.write(0, 0, '序号', style)
    sheet.write(0, 1, '城市省份', style)
    sheet.write(0, 2, '城市名', style)
    sheet.write(0, 3, '城市编码 ', style)
    sheet.write(0, 4, '城市的请求url', style)

    print len(city_list)

    for i in xrange(len(city_list)):
        sheet.write(i + 1, 0, i + 1, style)
        sheet.write(i + 1, 1, province_list[i], style)
        sheet.write(i + 1, 2, city_list[i], style)
        sheet.write(i + 1, 3, code_list[i], style)
        sheet.write(i + 1, 4, city_url_list[i], style)

    wbk.save(file_name)
    print ('数据写入完成')


if __name__ == '__main__':
    city_list = []
    province_list = []
    code_list = []
    city_url_list = []
    for i in xrange(54511, 59981):
        if i != 58151:
            print 'http://www.nmc.cn/f/rest/real/{}'.format(i)
            province, city, code, city_url = get_json_info('http://www.nmc.cn/f/rest/real/{}'.format(i))
            if province is not None:
                province_list.append(province)
                city_list.append(city)
                code_list.append(code)
                city_url_list.append(city_url)

    write_xls(city_list, province_list, code_list, city_url_list)

分析xls

第一步把拿到的xls进行分析,发现相关少了这几个省份<澳门香港黑龙江吉林台湾内蒙古新疆宁夏>
发现少了这几个省份，这个时候把range扩大

城市名	对应的城市ID
澳门	45011
香港	45005
黑龙江	45005
吉林	54172
台湾	58968
内蒙古	58968
新疆	51463
宁夏	53935

把数据插入本地数据库

因为我用到的是OTS，个人是把城市ID作为主键，再反查城市名

#! /usr/bin/env python
# -*- coding: utf-8 -*-

import lib.dao.weather as com_weather
import lib.common.func as com_func


def get_city_id_from_city_name(city_name):
    """
    根据城市名去查cityname
    :param city_name:
    :return:
    """
    all_city_info = com_weather.get_weather_list_cache()
    city_id = 0
    for city_info in eval(all_city_info):
        if city_info['city'] == city_name:
            print city_info['city']
            city_id = city_info['city_id']
            break
    return city_id


def get_city_id_list():
    city_id_list = []
    all_city_info = com_weather.get_weather_list_cache()
    print com_func.sort_serialize(all_city_info)

    for city_info in eval(all_city_info):
        city_id_list.append(city_info.get('city_id', 0))
    return city_id_list