Q:

How do I scrape the data only for a specific year and month?

I'm trying to scrape the review identifier and review date in MDY format (e.g. January 6, 2018) from trip advisor website. Since there are a lot of reviews even for one hotel, I'm trying to scrape the review data within a certain period of time only. Let's say that period of time ranges from June 2017 to December 2017, how do I sort out the corresponding data? The simplest thing I can think of is I scrape all the reviews for a hotel and then filter out what I want later. But the website is not letting me to scrape all the reviews (sometimes getting blocked). So my plan is to scrape only that portion of the reviews while not getting blocked. Could anyone have a good idea on how to scrape the data within June 2017 - December 2017?
Here's my current code. This gives 2 variables (review id, review date) with 5 observations that ranged from Dec 2017 to Jan 2018.
import datetime
import urllib
from urllib.request import urlopen
from bs4 import BeautifulSoup

listed_reviews_total=[]

#an example page where there are 5 reviews. I had multiple elements but showed you only one to be simple.
listing_url=["https://www.tripadvisor.com/Hotel_Review-g155032-d12237304-Reviews-or155-Holiday_Inn_Hotel_Suites_Montreal_Centre_ville_Ouest-Montreal_Quebec.html"]

for k in listing_url:
    page=urlopen(k)
    soup=BeautifulSoup(page,"html.parser")    
    time.sleep(3)
    listing=soup.findAll("div",{"class":"review-container"})
    for i in range(len(listing)):
            review_id=listing[i]["data-reviewid"]
            listing_v1=soup.findAll("div", {"class":"prw_rup prw_reviews_review_resp"})
            review_date=listing_v1[i].find("span",{"class":"ratingDate"})
            review_date_detail=review_date["title"]
            listed_reviews=[review_id, review_date_detail]
            listed_reviews.extend([k])
            listed_reviews_total.append(listed_reviews)
    print (len(listed_reviews_total))
    print (listed_reviews_total)

import requests
from bs4 import BeautifulSoup
import datetime
import time
import csv

#There are actually more than two but to put it simple.
listing_url=["https://www.tripadvisor.com/Hotel_Review-g155032-d12237304-Reviews-Holiday_Inn_Hotel_Suites_Montreal_Centre_ville_Ouest-Montreal_Quebec.html","https://www.tripadvisor.com/Hotel_Review-g155032-d155205-Reviews-Comfort_Suites_Downtown-Montreal_Quebec.html"]

for k in listing_url:
    data = 'changeSet=REVIEW_LIST'
    headers = {
        'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'cookie': 'VRMCID=%1%V1*id.10568*llp.%2F*e.1560821283491; TAUnique=%1%enc%3AlDqra346CYNe2oFFOebaM2LiS5PSM6Dpd5ss7m1Idn8%3D; CM=%1%PremiumMobSess%2C%2C-1%7Ct4b-pc%2C%2C-1%7CRestAds%2FRPers%2C%2C-1%7CRCPers%2C%2C-1%7CWShadeSeen%2C%2C-1%7CTheForkMCCPers%2C%2C-1%7CHomeASess%2C2%2C-1%7CPremiumSURPers%2C%2C-1%7CPremiumMCSess%2C%2C-1%7CRestPartSess%2C%2C-1%7CRestPremRSess%2C%2C-1%7CCCSess%2C%2C-1%7CCYLSess%2C%2C-1%7CViatorMCPers%2C%2C-1%7Csesssticker%2C%2C-1%7CPremiumORSess%2C%2C-1%7Ct4b-sc%2C%2C-1%7CRestAdsPers%2C%2C-1%7CMC_IB_UPSELL_IB_LOGOS2%2C%2C-1%7Cb2bmcpers%2C%2C-1%7CPremMCBtmSess%2C%2C-1%7CPremiumSURSess%2C%2C-1%7CMC_IB_UPSELL_IB_LOGOS%2C%2C-1%7CLaFourchette+Banners%2C%2C-1%7Csess_rev%2C%2C-1%7Csessamex%2C%2C-1%7CPremiumRRSess%2C%2C-1%7CTADORSess%2C%2C-1%7CTARSWBPers%2C%2C-1%7CSPMCSess%2C%2C-1%7CTheForkORSess%2C%2C-1%7CTheForkRRSess%2C%2C-1%7Cpers_rev%2C%2C-1%7CSPMCWBPers%2C%2C-1%7CRBAPers%2C%2C-1%7CRestAds%2FRSess%2C%2C-1%7CHomeAPers%2C%2C-1%7CPremiumMobPers%2C%2C-1%7CRCSess%2C%2C-1%7CLaFourchette+MC+Banners%2C%2C-1%7CRestAdsCCSess%2C%2C-1%7CRestPartPers%2C%2C-1%7CRestPremRPers%2C%2C-1%7Csh%2C%2C-1%7Cpssamex%2C%2C-1%7CTheForkMCCSess%2C%2C-1%7CCYLPers%2C%2C-1%7CCCPers%2C%2C-1%7Cb2bmcsess%2C%2C-1%7CSPMCPers%2C%2C-1%7CViatorMCSess%2C%2C-1%7CPremiumMCPers%2C%2C-1%7CPremiumRRPers%2C%2C-1%7CRestAdsCCPers%2C%2C-1%7CTADORPers%2C%2C-1%7CTheForkORPers%2C%2C-1%7CPremMCBtmPers%2C%2C-1%7CTheForkRRPers%2C%2C-1%7CTARSWBSess%2C%2C-1%7CPremiumORPers%2C%2C-1%7CRestAdsSess%2C%2C-1%7CRBASess%2C%2C-1%7CSPORPers%2C%2C-1%7Cperssticker%2C%2C-1%7CSPMCWBSess%2C%2C-1%7C; __gads=ID=80b73b494251f08f:T=1560216487:S=ALNI_MYxdo3gOkPGrsxb9kGdERgOvL-biw; TASSK=enc%3AAM85QGP7TErhmUIO9mM5yPlJa4z6Jx3T9zTXeFwj5yaDuxT1RvDvGCemE70Jbw5%2BtIBDquAnUNZ2m5VLGuuvRtGmkCtlMC1OW7e6AVSbk0xUPC7TnSsaw1U8FhrBl8t3%2BQ%3D%3D; BEPIN=%1%16ccacac52e%3Bbak207b.b.tripadvisor.com%3A10023%3B; ServerPool=T; PMC=V2*MS.3*MD.20190825*LD.20190825; TATravelInfo=V2*AY.2019*AM.9*AD.8*DY.2019*DM.9*DD.9*A.2*MG.-1*HP.2*FL.3*DSM.1566770382318*RS.1; TART=%1%enc%3AXtqBRTnm2jNrsrMi2CxcSuIYVpKWXuXqDscF5AFmxraBqxqvS4TTNL4xNEA4Fg2JNox8JbUSTxk%3D; PAC=AKI0Ak_wHxo1xVz2A_kIgU8afLzDybZKx16rKbTcC2dAWYfbE4f9VQqDxKMvap6yTTyaWhGH6TJKElQ-o5EQK6vSAux0awkZyssfXcMSzr_a4yoU-UxjSjh8VBCKr6rPgjUZapUSxllTx7dNRbU1KKzkv83-AR7D1uYIPHYGswVCierahneqD1HvcFWMn8_nzA%3D%3D; TAReturnTo=%1%%2FHotel_Review-g155032-d12237304-Reviews-Holiday_Inn_Hotel_Suites_Montreal_Centre_ville_Ouest-Montreal_Quebec.html; roybatty=TNI1625!ADHt%2BO1VDwozzKEvwcyfZrPDM9h5c5NXZX4eLIkKd6GrgAeBXdyPbE0DqaM%2FNvGtkS%2Fzr3yXhUr4R7rO%2FBLS6BU6%2FG9N2prFQs0s17Om2OPfFGHsV5JqdMlv6W2ykScrBajEJA%2FCnEcL01PxPoAfVCDgxWWLTtXSl2a18k6Bxl12%2C1; SRT=%1%enc%3AXtqBRTnm2jNrsrMi2CxcSuIYVpKWXuXqDscF5AFmxraBqxqvS4TTNL4xNEA4Fg2JNox8JbUSTxk%3D; TASession=V2ID.6F7A576FFF33B3D5E31756EE43C2D7AC*SQ.50*LS.PageMoniker*GR.75*TCPAR.95*TBR.16*EXEX.87*ABTR.83*PHTB.24*FS.84*CPU.94*HS.recommended*ES.popularity*DS.5*SAS.popularity*FPS.oldFirst*LF.en*FA.1*DF.0*TRA.false*LD.12237304; TAUD=LA-1566770383428-1*RDD-1-2019_08_25*HDD-1-2019_09_08.2019_09_09.1*LD-9055362-2019.9.8.2019.9.9*LG-9055364-2.1.F.',
        'x-requested-with': 'XMLHttpRequest'
    }

    #Date ranges for reviews
    start = datetime.date(2019, 7, 1)
    end = datetime.date(2019, 8, 25)

    s = requests.Session()
    #s.get('https://www.tripadvisor.com/Hotel_Review-g155032-d12237304-Reviews-Holiday_Inn_Hotel_Suites_Montreal_Centre_ville_Ouest-Montreal_Quebec.html')
    s.get(k)

    listed_reviews_total = []
    for next_num in range(0, 25, 5):

        url_1='-'.join(k.split('-',4)[:4])
        url_2='-'.join(k.split('-',4)[4:])
        middle="-or{%d}-" % next_num
        final_k=url_1+middle+url_2

    #r = s.post(f"https://www.tripadvisor.com/Hotel_Review-g155032-d12237304-Reviews-or{next_num}-Holiday_Inn_Hotel_Suites_Montreal_Centre_ville_Ouest-Montreal_Quebec.html", data=data, headers=headers)
    #r= s.post(f"https://www.tripadvisor.com/Hotel_Review-g155032-d155205-Reviews-or{next_num}-Comfort_Suites_Downtown-Montreal_Quebec.html", data=data, headers=headers)
        r= s.post(final_k, data=data, headers=headers)

        #Try to parse again in case that it didn't work properly.
        for try_again in range(0,10,1):    

            soup = BeautifulSoup(r.text, 'html.parser')
            time.sleep(4)
            listing=soup.find_all("div",{"class":"review-container"})

            if len(listing)==0:
                print ("fail")
                continue
            elif len(listing)!=0:
                print ("success")
                break

        for review in listing:
            review_id=review["data-reviewid"]
            listing_v1=soup.find_all("div", {"class":"prw_rup prw_reviews_review_resp"})
            review_date=review.find("span",{"class":"ratingDate"})
            review_date_detail=review_date["title"]
            date_obj = datetime.datetime.strptime(review_date_detail, '%B %d, %Y').date()
            if end-date_obj> datetime.timedelta(0) and date_obj-start>datetime.timedelta(0): #Checks if review is within specific date
                listed_reviews=[review_id, review_date_detail]
                listed_reviews.extend([k])
                listed_reviews_total.append(listed_reviews)

print(len(listed_reviews_total))
print(listed_reviews_total)

#Export 'listed_reviews_total' (each list will be at each row) to CSV directly (header included).
with open("trip_advisor_reviews_stack_overflow_v3.csv","w") as output:
    csv_out = csv.writer(output, lineterminator='\n')
    #csv_out.writerow(['review_id','review_date','rating','listing_url'])
    csv_out.writerow(['review_id','review_date','listing_url']) 
    for row in listed_reviews_total:
        csv_out.writerow(row)

A:

The method I used to look at all the reviews at this hotel is with the requests library. The url changes slightly every time you look at the next set of 5 reviews. I was able grab the ID and dates of all the reviews, but I still added the date functionality to sort out the corresponding data.
import requests
from bs4 import BeautifulSoup
import datetime

data = 'changeSet=REVIEW_LIST'
headers = {
    'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'cookie': 'VRMCID=%1%V1*id.10568*llp.%2F*e.1560821283491; TAUnique=%1%enc%3AlDqra346CYNe2oFFOebaM2LiS5PSM6Dpd5ss7m1Idn8%3D; CM=%1%PremiumMobSess%2C%2C-1%7Ct4b-pc%2C%2C-1%7CRestAds%2FRPers%2C%2C-1%7CRCPers%2C%2C-1%7CWShadeSeen%2C%2C-1%7CTheForkMCCPers%2C%2C-1%7CHomeASess%2C2%2C-1%7CPremiumSURPers%2C%2C-1%7CPremiumMCSess%2C%2C-1%7CRestPartSess%2C%2C-1%7CRestPremRSess%2C%2C-1%7CCCSess%2C%2C-1%7CCYLSess%2C%2C-1%7CViatorMCPers%2C%2C-1%7Csesssticker%2C%2C-1%7CPremiumORSess%2C%2C-1%7Ct4b-sc%2C%2C-1%7CRestAdsPers%2C%2C-1%7CMC_IB_UPSELL_IB_LOGOS2%2C%2C-1%7Cb2bmcpers%2C%2C-1%7CPremMCBtmSess%2C%2C-1%7CPremiumSURSess%2C%2C-1%7CMC_IB_UPSELL_IB_LOGOS%2C%2C-1%7CLaFourchette+Banners%2C%2C-1%7Csess_rev%2C%2C-1%7Csessamex%2C%2C-1%7CPremiumRRSess%2C%2C-1%7CTADORSess%2C%2C-1%7CTARSWBPers%2C%2C-1%7CSPMCSess%2C%2C-1%7CTheForkORSess%2C%2C-1%7CTheForkRRSess%2C%2C-1%7Cpers_rev%2C%2C-1%7CSPMCWBPers%2C%2C-1%7CRBAPers%2C%2C-1%7CRestAds%2FRSess%2C%2C-1%7CHomeAPers%2C%2C-1%7CPremiumMobPers%2C%2C-1%7CRCSess%2C%2C-1%7CLaFourchette+MC+Banners%2C%2C-1%7CRestAdsCCSess%2C%2C-1%7CRestPartPers%2C%2C-1%7CRestPremRPers%2C%2C-1%7Csh%2C%2C-1%7Cpssamex%2C%2C-1%7CTheForkMCCSess%2C%2C-1%7CCYLPers%2C%2C-1%7CCCPers%2C%2C-1%7Cb2bmcsess%2C%2C-1%7CSPMCPers%2C%2C-1%7CViatorMCSess%2C%2C-1%7CPremiumMCPers%2C%2C-1%7CPremiumRRPers%2C%2C-1%7CRestAdsCCPers%2C%2C-1%7CTADORPers%2C%2C-1%7CTheForkORPers%2C%2C-1%7CPremMCBtmPers%2C%2C-1%7CTheForkRRPers%2C%2C-1%7CTARSWBSess%2C%2C-1%7CPremiumORPers%2C%2C-1%7CRestAdsSess%2C%2C-1%7CRBASess%2C%2C-1%7CSPORPers%2C%2C-1%7Cperssticker%2C%2C-1%7CSPMCWBSess%2C%2C-1%7C; __gads=ID=80b73b494251f08f:T=1560216487:S=ALNI_MYxdo3gOkPGrsxb9kGdERgOvL-biw; TASSK=enc%3AAM85QGP7TErhmUIO9mM5yPlJa4z6Jx3T9zTXeFwj5yaDuxT1RvDvGCemE70Jbw5%2BtIBDquAnUNZ2m5VLGuuvRtGmkCtlMC1OW7e6AVSbk0xUPC7TnSsaw1U8FhrBl8t3%2BQ%3D%3D; BEPIN=%1%16ccacac52e%3Bbak207b.b.tripadvisor.com%3A10023%3B; ServerPool=T; PMC=V2*MS.3*MD.20190825*LD.20190825; TATravelInfo=V2*AY.2019*AM.9*AD.8*DY.2019*DM.9*DD.9*A.2*MG.-1*HP.2*FL.3*DSM.1566770382318*RS.1; TART=%1%enc%3AXtqBRTnm2jNrsrMi2CxcSuIYVpKWXuXqDscF5AFmxraBqxqvS4TTNL4xNEA4Fg2JNox8JbUSTxk%3D; PAC=AKI0Ak_wHxo1xVz2A_kIgU8afLzDybZKx16rKbTcC2dAWYfbE4f9VQqDxKMvap6yTTyaWhGH6TJKElQ-o5EQK6vSAux0awkZyssfXcMSzr_a4yoU-UxjSjh8VBCKr6rPgjUZapUSxllTx7dNRbU1KKzkv83-AR7D1uYIPHYGswVCierahneqD1HvcFWMn8_nzA%3D%3D; TAReturnTo=%1%%2FHotel_Review-g155032-d12237304-Reviews-Holiday_Inn_Hotel_Suites_Montreal_Centre_ville_Ouest-Montreal_Quebec.html; roybatty=TNI1625!ADHt%2BO1VDwozzKEvwcyfZrPDM9h5c5NXZX4eLIkKd6GrgAeBXdyPbE0DqaM%2FNvGtkS%2Fzr3yXhUr4R7rO%2FBLS6BU6%2FG9N2prFQs0s17Om2OPfFGHsV5JqdMlv6W2ykScrBajEJA%2FCnEcL01PxPoAfVCDgxWWLTtXSl2a18k6Bxl12%2C1; SRT=%1%enc%3AXtqBRTnm2jNrsrMi2CxcSuIYVpKWXuXqDscF5AFmxraBqxqvS4TTNL4xNEA4Fg2JNox8JbUSTxk%3D; TASession=V2ID.6F7A576FFF33B3D5E31756EE43C2D7AC*SQ.50*LS.PageMoniker*GR.75*TCPAR.95*TBR.16*EXEX.87*ABTR.83*PHTB.24*FS.84*CPU.94*HS.recommended*ES.popularity*DS.5*SAS.popularity*FPS.oldFirst*LF.en*FA.1*DF.0*TRA.false*LD.12237304; TAUD=LA-1566770383428-1*RDD-1-2019_08_25*HDD-1-2019_09_08.2019_09_09.1*LD-9055362-2019.9.8.2019.9.9*LG-9055364-2.1.F.',
    'x-requested-with': 'XMLHttpRequest'
}

#Date ranges for reviews
start = datetime.date(2017, 6, 1)
end = datetime.date(2017, 12, 31)

s = requests.Session()
s.get('https://www.tripadvisor.com/Hotel_Review-g155032-d12237304-Reviews-Holiday_Inn_Hotel_Suites_Montreal_Centre_ville_Ouest-Montreal_Quebec.html')

listed_reviews_total = []
for next_num in range(0, 211, 5):
    r = s.post(f"https://www.tripadvisor.com/Hotel_Review-g155032-d12237304-Reviews-or{next_num}-Holiday_Inn_Hotel_Suites_Montreal_Centre_ville_Ouest-Montreal_Quebec.html", data=data, headers=headers)
    soup = BeautifulSoup(r.text, 'html.parser')
    listing=soup.find_all("div",{"class":"review-container"})
    for review in listing:
        review_id=review["data-reviewid"]
        listing_v1=soup.find_all("div", {"class":"prw_rup prw_reviews_review_resp"})
        review_date=review.find("span",{"class":"ratingDate"})
        review_date_detail=review_date["title"]
        date_obj = datetime.datetime.strptime(review_date_detail, '%B %d, %Y').date()
        if end-date_obj> datetime.timedelta(0) and date_obj-start>datetime.timedelta(0): #Checks if review is within specific date
            listed_reviews_total.append([review_id, review_date_detail])

print(len(listed_reviews_total))
print(listed_reviews_total)

