当前位置：首页 > news >正文

Python爬取国家统计局最新行政区划代码和城乡划分代码

news 2025/5/16 6:43:41

Python爬取国家统计局最新行政区划代码和城乡划分代码

0、前言

国家数据，慎爬！！！

因业务开发需要获取最新的全国省市区行政区划代码，特整理此代码用于抓取最新的全国省市区行政区划代码。

1、数据来源

数据来源于 国家统计局2023年度全国统计用区划代码和城乡划分代码

在这里插入图片描述

2、python代码

import re
import urllib.request
import pandas as pdINDEX_URL = "https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/"id_counter = 1
all_data = []
province_id_map = {}
city_id_map = []def get_province_code():global id_counter# 获取省级初始页province_response = urllib.request.urlopen(INDEX_URL + "index.html").read().decode("utf-8")# 获取省份名称+城市初始页数据province_data = re.findall(r"<td><a href=\"(.*?)\">(.*?)<br /></a></td>", province_response)# 获取城市名称 + 省份代码 + 城市代码for url, name in province_data:# 省份代码，取前6位并补0province_code = url.replace(".html", "") + "000000"# 省级 level 为 1，parent_id 为 nullall_data.append([id_counter, name, province_code[:6], 1, "null"])print(f"省: {id_counter}, {name}, {province_code[:6]}, 1, null")  # 打印日志province_id_map[province_code[:6]] = id_counterprovince_id = id_counterid_counter += 1# 获取该省份下的城市和区县get_city_code(url, province_id, name)def get_city_code(province_url, province_id, province_name):global id_counter# 获取城市初始页city_response = urllib.request.urlopen(INDEX_URL + province_url).read().decode("utf-8")# 获取城市名称+城市code+地区初始页数据city_data = re.findall(r"<tr class=\"citytr\"><td><a href=\"(.*?)\">(.*?)</a></td><td><a href=\"(.*?)\">("r".*?)</a></td></tr>", city_response)# 如果是直辖市的情况（只有一个市级行政单位且包含“市辖区”）if len(city_data) == 1 and "市辖区" in city_data[0][3]:# 对于市辖区，使用省名称，但编码使用市辖区自身的编码city_id = id_counter# 市级 level 为 2all_data.append([city_id, province_name, city_data[0][1][:6], 2, province_id])  # 使用省名称和市辖区编码# print(f"市辖区处理，使用省名称: {province_name}, 但使用市辖区编码: {city_data[0][1][:6]}, 2, {province_id}")  # 打印日志print(f"市: {id_counter}, {province_name}, {city_data[0][1][:6]}, 2, {province_id}")  # 打印日志id_counter += 1# 获取区县get_area_code(city_data[0][0], city_id)else:# 正常处理市和区for url, city_code, _url, city_name in city_data:city_code = city_code[:6]  # 城市代码取前6位# 市级 level 为 2all_data.append([id_counter, city_name, city_code, 2, province_id])print(f"市: {id_counter}, {city_name}, {city_code}, 2, {province_id}")  # 打印日志city_id_map.append(city_code)city_id = id_counterid_counter += 1# 获取区县get_area_code(url, city_id)def get_area_code(city_url, city_id):global id_counter# 获取区县初始页area_response = urllib.request.urlopen(INDEX_URL + city_url).read().decode("utf-8")# 获取区县名称+区县code+街道初始页数据area_data = re.findall(r"<tr class=\"countytr\"><td><a href=\"(.*?)\">(.*?)</a></td><td><a href=\"(.*?)\">("r".*?)</a></td></tr>", area_response)for url, district_code, _url, district_name in area_data:district_code = district_code[:6]  # 区县代码取前6位# 区县 level 为 3all_data.append([id_counter, district_name, district_code, 3, city_id])print(f"区: {id_counter}, {district_name}, {district_code}, 3, {city_id}")  # 打印日志id_counter += 1def generate_sql(data):"""生成SQL语句并保存到sql.txt文件中:param data: 所有爬取的省市区数据"""with open("./省市区行政代码/sql.txt", "w", encoding="utf-8") as f:# 写建表语句f.write("""CREATE TABLE area (id INT PRIMARY KEY,name VARCHAR(255),code CHAR(6),level TINYINT,parent_id INT);""")# 写插入数据的SQL语句for row in data:id_, name, code, level, parent_id = rowinsert_sql = f"INSERT INTO area (id, name, code, level, parent_id) VALUES ({id_}, '{name}', '{code}', {level}, {parent_id if parent_id != 'null' else 'NULL'});\n"f.write(insert_sql)def main():"""主启动函数:return:"""get_province_code()columns = ['id', 'name', 'code', 'level', 'parent_id']df = pd.DataFrame(all_data, columns=columns)df.to_csv("./省市区行政代码/code.csv", index=False)# 生成SQL语句并输出到sql.txtgenerate_sql(all_data)if __name__ == "__main__":main()