攜程酒店信息采集

攜程酒店信息采集importurllib.requestfrombs4importBeautifulSoupimportcsvimportredefopenlink(link):maxTryNum=20fortriesinrange(maxTryNum):try:response=urllib.request.urlopen(link).read().decode('utf-8')returnresponseexcept:iftries<(maxTryNum-1):continueelse:print("Hastried%dtimestoaccessurl%s,allfailed!",maxTryNum,link)returnNonebreakcityurl='/jiudian/neimenggu'cityhtml=openlink(cityurl)citysoup=BeautifulSoup(cityhtml,'html.parser')city_list=citysoup.find_all('ul',attrs={"class":"p_n_listgrid_8"})citysoup1=BeautifulSoup(str(city_list),'html.parser')city_list1=citysoup1.find_all('a',attrs={"href":True})city_id=[]forcityincity_list1:city_id1=city['href']city_id.append(city_id1)print(city_id)forcityidincity_id[13:14]:cityname=re.findall(r"/([a-z]+?)[0-9]",cityid)csvFile2=open('Hotel'+str(cityname)+'.csv','w',newline='',encoding='utf-8-sig')#設置newline，否則兩?之間會空??writer=csv.writer(csvFile2)data1=['所屬盟市','所屬旗縣','酒店名稱','星級','房間數(shù)','開業(yè)年份','地址','聯(lián)系?式']data=[data1]m=len(data)foriinrange(m):writer.writerow(data[i])print('programbegin')pagecounturl=''+cityidprint(pagecounturl)pagecounthtml=openlink(pagecounturl)#1.獲取酒店名稱信息pagecountsoup=BeautifulSoup(pagecounthtml,'html.parser')page_count=pagecountsoup.find_all(attrs={"datacount":True})forcountinpage_count:pagecount=count['datacount']print(pagecount)forkinrange(1,int(pagecount)+1):print("***正在抓取第"+str(k)+"頁********")pageurl=''hotel_id=[]pageurl=''+cityid+'/p'+str(k)print(pageurl)paghtml=openlink(pageurl)#1.獲取酒店名稱信息ifpaghtml==None:withopen('Pageerror.txt','a+')asf:f.write(pageurl+'\n')continuepagesoup=BeautifulSoup(paghtml,'html.parser')hotel_list=pagesoup.find_all("h2",attrs={"data-id":True})forhotelinhotel_list:hotel_id1=hotel['data-id']hotel_id.append(hotel_id1)print("hotel_id是:")print(hotel_id,len(hotel_id)+1)result=[]foriinrange(1,len(hotel_id)+1):url1=''result0=[]#url1='/hotel/'+hotel_id[i-1]+'.html?isFull=F#ctm_ref=hod_sr_lst_dl_n_'+str(k)+'_'+str(i)#為爬取的當前頁?的?址，#num[i]為每個頁??址id，'.html?isFull=F#ctm_ref=hod_sr_lst_dl_n_2_'需要修改2為3，4，等等，#str（i）是個數(shù)url1='/hotel/'+hotel_id[i-1]+'.html?isFull=F'print(url1)html1=openlink(url1)#1.獲取酒店名稱信息ifhtml1==None:withopen('Hotelerror.txt','a+')asf1:f.write(url1+'\n')continuesoup1=BeautifulSoup(html1,'html.parser')print("soup1是：")result1=soup1.find_all('h1')print("result1是：")print(result1)hotelName=result1[0].stringprint("酒店名稱為:{}".format(hotelName))print(i)#2.獲取酒店位置soup12=BeautifulSoup(html1,'html.parser')result12=soup12.find_all(attrs={"class":"adress"})hotelAdress_m=result12[0].get_text().split('\n')[1]hotelAdress_q=result12[0].get_text().split('\n')[2]hotelAdress_a=result12[0].get_text().split('\n')[3]print("酒店所屬盟市為:{}".format(hotelAdress_m))print("酒店所屬旗縣為:{}".format(hotelAdress_q))print("酒店地址為:{}".format(hotelAdress_a))#3.獲取酒店房間數(shù)soup13=BeautifulSoup(html1,'html.parser')result13=soup13.find_all(attrs={"class":"htl_room_txttext_3l"})result13=str(result13)soup14=BeautifulSoup(result13,'html.parser')result14=soup14.find_all('p')print(result14[0].get_text().strip().split('\n')[0].split('\xa0\xa0'))hotel_inform=result14[0].get_text().strip().split('\n')[0]regex_str1=".*?([0-9]+年開業(yè))"regex_str2=".*?([0-9]+間房)"match_obj1=re.match(regex_str1,str(hotel_inform))match_obj2=re.match(regex_str2,str(hotel_inform))ifmatch_obj1:hotelStartbussi=match_obj1.group(1)[:-3]print("酒店開業(yè)年份為:{}".format(hotelStartbussi))else:hotelStartbussi="?"print("酒店開業(yè)年份為:{}".format(hotelStartbussi))match_ob2=re.match(regex_str2,str(hotel_inform))ifmatch_obj2:ifmatch_obj2:hotelRoomnum=match_obj2.group(1)[:-2]print("酒店房間數(shù)為:{}".format(hotelRoomnum))else:hotelRoomnum="?"print("酒店開業(yè)年份為:{}".format(hotelStartbussi))#4.獲取酒店床位數(shù)#5.獲取酒店電話result15=soup14.find_all('span')if'data-real'instr(result15[0]):hotelContacter=result15[0]['data-real'].split('<a')[0].strip().split('')[0][2:]print("酒店聯(lián)系?式為:{}".format(hotelContacter))else:hotelContacter='?'print("酒店聯(lián)系?式為:{}".format(hotelContacter))#6.星級soup16=BeautifulSoup(html1,'html.parser')result16=soup16.find_all(attrs={"class":"grade"})result16=str(result16)soup17=BeautifulSoup(result16,'html.parser')result17=soup17.find_all('span')if'title'instr(result17[0]):hotelStar=result17[0]['title']print("酒店星級為:{}".format(hotelStar))else:hotelStar='?'print("酒店星級為:{}".format(hotelStar))result0=[format(hotelAdress_m),format(hotelAdress_q),format(hotelName),fo