大乐透

爬取1

# 爬取大乐透的开奖历史数据# http://www.lottery.gov.cn/api/lottery_kj_detail_new.jspx?_ltype=4&_term=19026import requestsimport reimport csvagent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'headers = {    'User-Agent': agent,}proxy={    "http":"125.39.9.34:9000",}url = 'http://www.lottery.gov.cn/api/lottery_kj_detail_new.jspx'start = int(input('输入开始期号：'))  # 18134end = int(input('输入结束期号：')) # 19029lottery_li = [] for qihao in range(start,end+1):    data={        '_ltype':'4',        '_term':qihao,    }    page_text = requests.post(url=url,headers=headers,data=data,proxies=proxy).text    print(page_text)    if page_text:        # 根据返回数据解析        # 开奖号码        lottery_data = re.findall('codeNumber\"\:\[(.*?)\],\"',page_text,re.M)        if lottery_data:            num_data = lottery_data[0].replace("\"",'')            # print(num_data) # 10,12,15,17,19,02,03            lottery_list = num_data.split(',')            lottery_list.insert(0,qihao)            # print(lottery_list) # ['10', '12', '15', '17', '19', '02', '03']            lottery_li.append(lottery_list)with open('lottery_data.csv','w',newline='') as csvf:    spanwriter=csv.writer(csvf,dialect='excel')   #创建writer对象    spanwriter.writerow(['qihao','red1','red2','red3','red4','red5','blue1','blue2'])  #使用writer的方法writerow写入到文件    spanwriter.writerows(lottery_li)  #迭代写入数据    print('done.....................')

爬取2（所有开奖记录）

# 爬取大乐透的开奖历史数据# http://www.lottery.gov.cn/historykj/history.jspx?_ltype=dltimport requestsimport reimport csvfrom lxml import etreeimport randomimport timeagent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'headers = {    'User-Agent': agent,}proxies=[    {"http":"125.39.9.34:9000"},    {"http":"222.139.125.232:8060"},]proxy = random.choice(proxies)params={    '_ltype':'dlt',}page = int(input("end page no:"))lottery_data = []for page_no in range(1,page+1):    url = 'http://www.lottery.gov.cn/historykj/history_%s.jspx' % page_no    page_text = requests.get(url=url,params=params,headers=headers,proxies=proxy).text    time.sleep(1)    # print(page_text)    tree = etree.HTML(page_text)    tr_list = tree.xpath('//div[@class="result"]/table/tbody/tr') # 每页所有的tr 20    td_list = tree.xpath('//div[@class="result"]/table/tbody/tr/td') # 每页所有的td 400        for num_tr in range(1,len(tr_list) + 1):    #     print(num_tr)        td_qihao = tree.xpath('//div[@class="result"]/table/tbody/tr[%s]/td[1]//text()'%(num_tr))        td_red1 = tree.xpath('//div[@class="result"]/table/tbody/tr[%s]/td[2]//text()'%(num_tr))        td_red2 = tree.xpath('//div[@class="result"]/table/tbody/tr[%s]/td[3]//text()'%(num_tr))        td_red3 = tree.xpath('//div[@class="result"]/table/tbody/tr[%s]/td[4]//text()'%(num_tr))        td_red4 = tree.xpath('//div[@class="result"]/table/tbody/tr[%s]/td[5]//text()'%(num_tr))        td_red5 = tree.xpath('//div[@class="result"]/table/tbody/tr[%s]/td[6]//text()'%(num_tr))        td_blue1 = tree.xpath('//div[@class="result"]/table/tbody/tr[%s]/td[7]//text()'%(num_tr))        td_blue2 = tree.xpath('//div[@class="result"]/table/tbody/tr[%s]/td[8]//text()'%(num_tr))        lottery_one = td_qihao + td_red1+ td_red2+td_red3+td_red4+td_red5+td_blue1+td_blue2        lottery_data.append(lottery_one)# 写入csvwith open('all_lottery.csv','w',newline='') as csvf:    spanwriter=csv.writer(csvf,dialect='excel')   #创建writer对象    spanwriter.writerow(['qihao','red1','red2','red3','red4','red5','blue1','blue2'])  #使用writer的方法writerow写入到文件    spanwriter.writerows(lottery_data)  #迭代写入数据      print('done..................................')