大部分源码来自于：/MG_ApinG/article/details/82992913，谢谢作者
首先需要安装requests模块：
Windows安装： pip install requests
Linux安装：pip install requests
源码如下：
# -*- coding: GBK -*-# author：zjp# author：fudl /09/13"""原版来自于网络：/MG_ApinG/article/details/82992913通过国家统计局官网获取中国所有城市数据http://www./tjsj/tjbz/tjyqhdmhcxhfdm//抓取后的数据格式：行政区划代码上级行政区划代码行政区划名称行政区划全路径级别（国为1级）110106000000110100000000丰台区北京市|市辖区|丰台区4"""import reimport requestsimport timeimport sysfileSavePath = 'E://data/China_Province_.txt' # 数据储存路径fileSavePath2 = 'E://data/China_Province__mistake.txt' # 错误信息储存路径code1 = []results1 = []code2 = []results2 = []code3 = []results3 = []code4 = []results4 = []code5 = []results5 = []Dates1 = []code2Map = dict()code3Map = dict()code4Map = dict()code5Map = dict()n = 0# 获取一级城市信息url = 'http://www./tjsj/tjbz/tjyqhdmhcxhfdm//index.html'response = requests.get(url)response.raise_for_status() # 如果 HTTP 请求返回了不成功的状态码,Response.raise_for_status() 会抛出一个 HTTPError异常response.encoding = "GBK" # response.apparent_encoding从内容中分析出的响应内容编码方式pattern = pile("<a href='(.*?)'>(.*?)<")responseResult = list(set(re.findall(pattern, response.text))) # 从首层页面获取进入第二层页面的html# print('result1')codes = []codesNameDist = dict()result1 = []# 获取所有省份及代码for result in responseResult:code = result[0].split('.')[0]codesNameDist[code] = resultcodes.append(code)# 排序省份codes.sort()# 按照顺序抓取for code in codes:result1.append(codesNameDist.get(code))# 从一级城市获取二级城市信息for cycle1 in range(len(result1)):try:address1_1 = result1[cycle1][0]code = address1_1.split(".")[0]# 如果上次卡住了重新运行时从上次卡住的省份开始，已经抓取的省份就不再抓取了，如下，已经抓取了北京市就<12if int(code) < 12:continuecode1 = address1_1.split(".")[0] + '0000000000'address1 = result1[cycle1][1] # 一级城市print('开始抓取-------->：', address1)# 1级到省with open(fileSavePath, 'a', encoding='GBK')as f:f.write(code1)f.write('\t')f.write('100000000000')f.write('\t')f.write(address1)f.write('\t')f.write(address1)f.write('\t')f.write('2')f.write('\n')f.close()url1 = 'http://www./tjsj/tjbz/tjyqhdmhcxhfdm//{0}'.format(result1[cycle1][0]) # 一级城市urlresponse1 = requests.get(url1)response1.raise_for_status()response1.encoding = "GBK"response1.close()pattern1 = pile("<a href='(.*?)'>(.*?)<") # 正则表达式提取目标字段result2_1 = list(set(re.findall(pattern1, response1.text)))result2 = []# print("result2_1", result2_1)for result in result2_1: # 爬取的城市信息和城市代码混在一起，需要将代码清除if '0' not in result[1]:result2.append(result)else:code2Map[result[0]] = result[1]except:print("Unexpected error:", sys.exc_info())with open(fileSavePath, 'a', encoding='GBK')as f:f.write('{0}|一级错误|一级错误|一级错误|{1}\n'.format('xd', sys.exc_info()))f.close()time.sleep(10)continue# 从二级城市获取三级城市信息for cycle2 in range(len(result2)):try:address2_1 = result2[cycle2][0] # 二级城市address2 = result2[cycle2][1] # 二级城市code2 = code2Map.get(result2[cycle2][0])# 2级到市address = '{0}|{1}'.format(address1, address2)with open(fileSavePath, 'a', encoding='GBK')as f:f.write(code2)f.write('\t')f.write(code1)f.write('\t')f.write(address2)f.write('\t')f.write(address)f.write('\t')f.write('3')f.write('\n')f.close()url2 = 'http://www./tjsj/tjbz/tjyqhdmhcxhfdm//{0}'.format(result2[cycle2][0]) # 二级城市url# print(url2)response2 = requests.get(url2)response2.raise_for_status()response2.encoding = "GBK"response2.close()pattern2 = pile("<a href='(.*?)'>(.*?)<")result3_1 = list(set(re.findall(pattern2, response2.text)))result3 = []for result in result3_1:if '0' not in result[1]:result3.append(result)else:code3Map[result[0]] = result[1]except:print("Unexpected error:", sys.exc_info())with open(fileSavePath, 'a', encoding='GBK') as f:f.write('{0}|二级错误|二级错误|二级错误|{1}\n'.format(address1, sys.exc_info()))f.close()time.sleep(10)continue# 从三级城市获取四级城市信息for cycle3 in range(len(result3)):try:code3 = code3Map.get(result3[cycle3][0])address3 = result3[cycle3][1] # 三级城市# 3级到区、县address = '{0}|{1}|{2}'.format(address1, address2, address3)with open(fileSavePath, 'a', encoding='GBK')as f:f.write(code3)f.write('\t')f.write(code2)f.write('\t')f.write(address3)f.write('\t')f.write(address)f.write('\t')f.write('4')f.write('\n')f.close()url3 = 'http://www./tjsj/tjbz/tjyqhdmhcxhfdm//{0}/{1}' \.format(result3[cycle3][0][3:5], result3[cycle3][0]) # 三级城市urlresponse3 = requests.get(url3)response3.raise_for_status()response3.encoding = "GBK"response3.close()pattern3 = pile("<a href='(.*?)'>(.*?)<")result4_1 = list(set(re.findall(pattern3, response3.text)))result4 = []for result in result4_1:if '0' not in result[1]:result4.append(result)else:code4Map[result[0]] = result[1]except:print("Unexpected error:", sys.exc_info())with open(fileSavePath, 'a', encoding='GBK')as f:f.write('{0}|三级错误|三级错误|三级错误|{1}\n'.format(address2, sys.exc_info()))f.close()time.sleep(10)continue# 从四级城市获取四五级城市信息for cycle4 in range(len(result4)):try:code4 = code4Map.get(result4[cycle4][0])address4 = result4[cycle4][1] # 四级城市# 4级到镇、街道address = '{0}|{1}|{2}|{3}'.format(address1, address2, address3, address4)with open(fileSavePath, 'a', encoding='GBK')as f:f.write(code4)f.write('\t')f.write(code3)f.write('\t')f.write(address4)f.write('\t')f.write(address)f.write('\t')f.write('5')f.write('\n')f.close()url4 = 'http://www./tjsj/tjbz/tjyqhdmhcxhfdm//{0}/{1}/{2}' \.format(result4[cycle4][0][3:5], result4[cycle4][0][5:7], result4[cycle4][0]) # 四级城市urlresponse4 = requests.get(url4)response4.raise_for_status()response4.encoding = "GBK"response4.close()time.sleep(1)pattern4_1 = pile("<a href='(.*?)'>(.*?)<")# 可能已经到最后一层pattern4 = pile("<tr class='villagetr'><td>(.*?)</td><td>.*?</td><td>(.*?)</td></tr>")result5_1 = list(set(re.findall(pattern4, response4.text)))result5 = []for result in result5_1:if '0' not in result[1]:result5.append(result[1])code5Map[result[1]] = result[0]if result5:for cycle5 in result5:# 5级到村address = '{0}|{1}|{2}|{3}|{4}'.format(address1, address2, address3, address4, cycle5)code5 = code5Map[cycle5]with open(fileSavePath, 'a', encoding='GBK')as f:f.write(code5)f.write('\t')f.write(code4)f.write('\t')f.write(cycle5)f.write('\t')f.write(address)f.write('\t')f.write('6')f.write('\n')f.close()else:print('此处第五层城市结果为空,网址为: {0}'.format(url4))with open(fileSavePath2, 'a', encoding='GBK')as f2:f2.write('此处第五层城市结果为空,网址为: {0}'.format(url4))f2.close()except:print("Unexpected error:", sys.exc_info())with open(fileSavePath, 'a', encoding='GBK')as f:f.write('{0}|四级错误|四级错误|四级错误|{1}\n'.format(address3, sys.exc_info()))f.close()time.sleep(10)continueprint('抓取结束==============>：', address1)print('well_done')
