我正在尝试抓取一个特定站点 ( https://chartink.com/screener/test-121377 ) 我正在尝试访问 URL 中提供的表。该表可以是动态形式。该站点启用了 JS,因此尝试 HTML 抓取是多余的。接下来,我尝试使用 cURL 访问 XHR 元素,它确实有效。我使用了以下代码:
import requests
headers = {
'authority': 'chartink.com',
'pragma': 'no-cache',
'cache-control': 'no-cache',
'accept': 'application/json, text/javascript, */*; q=0.01',
'x-csrf-token': 'HYMZ6XnOySvn1w4fHEN0Hu1ublftvidgdBD5qywR',
'x-requested-with': 'XMLHttpRequest',
'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36',
'sentry-trace': 'd9b8f7ffcae445f7a74913d5ac143918-9fa7556ebb76186c-0',
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'origin': 'https://chartink.com',
'sec-fetch-site': 'same-origin',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://chartink.com/screener/test-121377',
'accept-language': 'en-IN,en;q=0.9,hi-IN;q=0.8,hi;q=0.7,en-GB;q=0.6,en-US;q=0.5',
'cookie': '_ga=GA1.2.468738165.1600680012; remember_web_59ba36addc2b2f9401580f014c7f58ea4e30989d=eyJpdiI6Inp6YWdERUJHSGxoWmhRaE00MGhONHc9PSIsInZhbHVlIjoiSXN1d09PXC9USmpGb2lFR21naFd1Q3RlcCtKSDdUXC8zdGRnMkRoQmNcL2loXC9tT0tMXC9sc0JtallJVCtNczVhVjhIdGUyb0xqMTIydlc0RWtQYVZcLzF1aFd3aXRPM0M3MDVObUtNK1FtdmJtZjY1YUxNbTlyTHhNWFhjU3VZNXpUTWY0SjZ4M1o1VEcydk1DWDY5YWVSdWRjbVwvQmkyYmlDT2VMWFVCM043Y2JIY21UTjNsdnJLaE1qSnNybmM2WVhISyIsIm1hYyI6IjhlNjBiZWUxMWU3NTBkOTRjM2NlYTZiNGI3YzJhYjg4M2E0OWI2MDU3ZDZkYWUzNzE1YjYxOTRhMTllMjY2ZWQifQ%3D%3D; __utmc=102564947; PHPSESSID=e3797d9e58f9f56b0307e7840f6839f7; bfp_sn_rf_8b2087b102c9e3e5ffed1c1478ed8b78=Direct/External; bfp_sn_rt_8b2087b102c9e3e5ffed1c1478ed8b78=1601299430046; bafp=d8afd2b0-018d-11eb-b966-a1985446bf25; hbcm_sd=1%7C1602315380999; _gid=GA1.2.729526333.1602491131; __utma=102564947.468738165.1600680012.1601134509.1602499714.2; __utmz=102564947.1602499714.2.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); XSRF-TOKEN=eyJpdiI6InVXSEFpN2xiYWZqWDkzNEdWZVBTSkE9PSIsInZhbHVlIjoiQlM2eVBYMjdYR3N4QTh6YmtFWmNoTUtOVHBIOFVoUG04WTdcL05iUm4yZExQR3ZucHlPVHhucVwvbjlHTW9VcmZCIiwibWFjIjoiMWEyMzViZDQ5YWI1YzIzY2UzYTdkNzRlN2UyZmYzNmY1ZWNkYTBmZmJmOWI2YjZmOGRmN2U2Y2IzOGE5ZmQwNiJ9; ci_session=eyJpdiI6ImtcLzNBOGxOb1l2bFlyOEFnQ2dxVHNBPT0iLCJ2YWx1ZSI6IjRpNVhXOWtGZlV0TWdwZlhIVGRmWFgrcGFyblwveGlkdVU5aHJraVdvbFROOURUa2Zhc2pcLzJNOE9qa0FwcjlwbSIsIm1hYyI6IjVlYTUwNDgxN2U0NTU4OWE0NTgxOTMwNjNjMzdhM2Q5MjRiMjJjMTBkYzI0ZTdkOTg1YTg5YmIxNTM3NzFlMDQifQ%3D%3D',}
data = …Run Code Online (Sandbox Code Playgroud)