Niftyindices.com not able to scrape Index data

Question

I was able to scrape data from NSE India's Niftyindices.com website earlier. this is the historical Index Data I need. The website is www.niftyindices.com and the historical data can be found under Reports Section of the home page under Historical Data. On this page you need to Select Index Type as Equity and the Index as NIFTY 100 Data range can be - i generally select from 01-Jan-2001 till today. till yesterday, i was able to fetch data from this site. but all of a sudden from today morning, there is no data appearing using the same code. following is the code i am using for scraping. bm_sv is an essential cookie which needs to be passed to the site - this is from my previous experience. The Json received does not have any data anymore.

Thanks for any help in advance

import requests
import json
from datetime import datetime


print('start')

data=[]
headers = {'Content-Type': 'application/json; charset=utf-8'
           ,'Accept':'application/json, text/javascript, */*; q=0.01'
           ,'Accept-Encoding':'gzip, deflate'
           ,'Accept-Language':'en-US,en;q=0.9'
           ,'Content-Length':'100'
           ,'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'
           ,'Cookie':'bm_sv=9B706239B47F50CA0B651E20BA5CBF74~YAAQFjkgF2zWSiyOAQAAdV0dQRfVSICkZc20SfI+PnDk8taK1Ppu1ZSmjclFkHqVgsGOE0vK3WnPMHuhY5kOStjVm4OnN1wm9SBRO3nIAvXWAVCR8iN23B8R7kHpcme82M8ytCrJ/LozntCxQlQSFqzuFwLw4+ZPBjdkICfQH4piCmjvZB3AH8NvCmf+nbzT34Q4JO4zYeYadkjlKjVRVIh0lzX2BK8crljTE9W+F1DUdtZYBRBUCM83OIfmZhnH6PnDu79C~1'
           }

row=['indexname','date','open','high','low','close']
data.append(row)

payload={'name': 'NIFTY 100','startDate':'01-Jan-2001','endDate': '10-Mar-2024'}
JSonURL='https://www.niftyindices.com/Backpage.aspx/getHistoricaldatatabletoString'
r=requests.post(JSonURL, data=json.dumps(payload),headers=headers)
print(r.text)
text=r.json()
print(text)
datab=json.loads(text['d'])
sorted_data=sorted(datab,key=lambda x: datetime.strptime(x['HistoricalDate'], '%d %b %Y'), reverse=False)
print('startdata available from: ',datetime.strftime(datetime.strptime(sorted_data[0]['HistoricalDate'], '%d %b %Y'),'%d-%b-%Y'))
print('data available till',datetime.strftime(datetime.strptime(sorted_data[len(datab)-1]['HistoricalDate'], '%d %b %Y'),'%d-%b-%Y\n'))
for rec in sorted_data:
    row=[]
    row.append(rec['Index Name'])
    row.append(datetime.strptime(rec['HistoricalDate'], '%d %b %Y'))
    row.append(rec['OPEN'].replace('-','0'))
    row.append(rec['HIGH'].replace('-','0'))
    row.append(rec['LOW'].replace('-','0'))
    row.append(rec['CLOSE'])
    print(row)
    data.append(row)
        
print(data)

Answer 1

The payloads used by you has to be nested dictionary. Kindly check the following (modified) code. This is working now!

1 Line added

mypayload={"cinfo":json.dumps(payload)}

1 Modified Line (using newly created payload as shown above)

r=requests.post(JSonURL, data=json.dumps(mypayload),headers=headers)

import requests
import json
from datetime import datetime


print('start')

data=[]
headers = {'Content-Type': 'application/json; charset=utf-8'
           ,'Accept':'application/json, text/javascript, */*; q=0.01'
           ,'Accept-Encoding':'gzip, deflate'
           ,'Accept-Language':'en-US,en;q=0.9'
           ,'Content-Length':'100'
           ,'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'
           ,'Cookie':'bm_sv=9B706239B47F50CA0B651E20BA5CBF74~YAAQFjkgF2zWSiyOAQAAdV0dQRfVSICkZc20SfI+PnDk8taK1Ppu1ZSmjclFkHqVgsGOE0vK3WnPMHuhY5kOStjVm4OnN1wm9SBRO3nIAvXWAVCR8iN23B8R7kHpcme82M8ytCrJ/LozntCxQlQSFqzuFwLw4+ZPBjdkICfQH4piCmjvZB3AH8NvCmf+nbzT34Q4JO4zYeYadkjlKjVRVIh0lzX2BK8crljTE9W+F1DUdtZYBRBUCM83OIfmZhnH6PnDu79C~1'
           }

row=['indexname','date','open','high','low','close']
data.append(row)

payload={'name': 'NIFTY 100','startDate':'01-Jan-2001','endDate': '10-Mar-2024'}
mypayload={"cinfo":json.dumps(payload)}
JSonURL='https://www.niftyindices.com/Backpage.aspx/getHistoricaldatatabletoString'
r=requests.post(JSonURL, data=json.dumps(mypayload),headers=headers)
print(r.text)
text=r.json()
print(text)
datab=json.loads(text['d'])
sorted_data=sorted(datab,key=lambda x: datetime.strptime(x['HistoricalDate'], '%d %b %Y'), reverse=False)
print('startdata available from: ',datetime.strftime(datetime.strptime(sorted_data[0]['HistoricalDate'], '%d %b %Y'),'%d-%b-%Y'))
print('data available till',datetime.strftime(datetime.strptime(sorted_data[len(datab)-1]['HistoricalDate'], '%d %b %Y'),'%d-%b-%Y\n'))
for rec in sorted_data:
    row=[]
    row.append(rec['Index Name'])
    row.append(datetime.strptime(rec['HistoricalDate'], '%d %b %Y'))
    row.append(rec['OPEN'].replace('-','0'))
    row.append(rec['HIGH'].replace('-','0'))
    row.append(rec['LOW'].replace('-','0'))
    row.append(rec['CLOSE'])
    print(row)
    data.append(row)
        
print(data)