I am trying to write a perl script that uses WWW::Mechanize::Chrome to determine whether or not a particular product is in stock on a product page (https://www.tenthousand.cc/products/interval-short) by clicking the appropriate radio buttons (color, size, inseam, liner), and then reading the value of the button (either "add to bag" or "sold out"). The page does not load properly without JS, otherwise I would just use WWW::Mechanize.
Here is my script so far:
> ./www_mechanize_chrome_testcase.pl
$VAR1 = [
{
'method' => 'Network.requestWillBeSent',
'params' => {
'loaderId' => 'EF16DAA4B81ECCB0A2A3FB76F0B0F
+3B6',
'requestId' => 'EF16DAA4B81ECCB0A2A3FB76F0B0
+F3B6',
'timestamp' => '1492087.342068',
'wallTime' => '1610320199.63641',
'type' => 'Document',
'redirectResponse' => {
'connectionReused' =
+> bless( do{\(my $o = 1)}, 'JSON::PP::Boolean' ),
'protocol' => 'h2',
'fromServiceWorker'
+=> bless( do{\(my $o = 0)}, 'JSON::PP::Boolean' ),
'url' => 'https://pi
+xel.tapad.com/idsync/ex/push/check?partner_id=2884&partner_url=https%
+3A%2F%2Ftr.snapchat.com%2Fcm%2Fp%3Frand%3D1610201931907%26pnid%3D140%
+26pcid%3D%24%7BTA_DEVICE_ID%7D',
'connectionId' => 80
+1,
'mimeType' => '',
'remoteIPAddress' =>
+ '107.178.246.49',
'headers' => {
'loca
+tion' => 'https://tr.snapchat.com/cm/p?rand=1610201931907&pnid=140&pc
+id=f5cb7c21-5398-11eb-9bb4-ba2c2ef2346d',
'stri
+ct-transport-security' => 'max-age=31536000',
'date
+' => 'Sun, 10 Jan 2021 23:09:58 GMT',
'alt-
+svc' => 'clear',
'via'
+ => '1.1 google',
'p3p'
+ => 'policyref="http://tapad-taptags.s3.amazonaws.com/policy/p3p.xml"
+, CP="NOI DSP COR ADM PSAo PSDo OURo SAMo UNRo OTRo BUS COM NAV DEM S
+TA PRE"',
'serv
+er' => 'Jetty(9.4.28.v20200408)',
'set-
+cookie' => 'TapAd_TS=1610320198882;Expires=Thu, 11 Mar 2021 23:09:58
+GMT;Path=/;Domain=.tapad.com;Secure;SameSite=None
TapAd_DID=f5cb7c21-5398-11eb-9bb4-ba2c2ef2346d;Expires=Thu, 11 Mar 202
+1 23:09:58 GMT;Path=/;Domain=.tapad.com;Secure;SameSite=None
TapAd_3WAY_SYNCS=;Expires=Thu, 11 Mar 2021 23:09:58 GMT;Path=/;Domain=
+.tapad.com;Secure;SameSite=None',
'cont
+ent-length' => '0'
},
'encodedDataLength'
+=> 376,
'securityState' => '
+secure',
'responseTime' => '1
+610320199635.73',
'fromPrefetchCache'
+=> $VAR1->[0]{'params'}{'redirectResponse'}{'fromServiceWorker'},
'fromDiskCache' => $
+VAR1->[0]{'params'}{'redirectResponse'}{'fromServiceWorker'},
'remotePort' => 443,
'requestHeaders' =>
+{
+ ':scheme' => 'https',
+ 'referer' => 'https://tr.snapchat.com/cm/i?pid=a8067835-6f53-4ec6-b
+1d7-509ec7db92f4',
+ 'upgrade-insecure-requests' => '1',
+ 'sec-fetch-mode' => 'navigate',
+ 'cookie' => 'TapAd_TS=1610320198882; TapAd_DID=f5cb7c21-5398-11eb-9
+bb4-ba2c2ef2346d',
+ 'accept-language' => 'en-US',
+ ':authority' => 'pixel.tapad.com',
+ ':path' => '/idsync/ex/push/check?partner_id=2884&partner_url=https
+%3A%2F%2Ftr.snapchat.com%2Fcm%2Fp%3Frand%3D1610201931907%26pnid%3D140
+%26pcid%3D%24%7BTA_DEVICE_ID%7D',
+ ':method' => 'GET',
+ 'sec-fetch-site' => 'cross-site',
+ 'accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,
+image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchang
+e;v=b3;q=0.9',
+ 'sec-fetch-dest' => 'iframe',
+ 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWeb
+Kit/537.36 (KHTML, like Gecko) HeadlessChrome/87.0.4280.88 Safari/537
+.36',
+ 'accept-encoding' => 'gzip, deflate, br'
+},
'status' => 302,
'statusText' => '',
'securityDetails' =>
+ {
+ 'subjectName' => '*.tapad.com',
+ 'cipher' => 'AES_128_GCM',
+ 'validTo' => 1636156800,
+ 'protocol' => 'TLS 1.3',
+ 'keyExchangeGroup' => 'X25519',
+ 'signedCertificateTimestampList' => [],
+ 'keyExchange' => '',
+ 'certificateId' => 0,
+ 'certificateTransparencyCompliance' => 'unknown',
+ 'sanList' => [
+ '*.tapad.com',
+ 'tapad.com'
+ ],
+ 'issuer' => 'DigiCert SHA2 Secure Server CA',
+ 'validFrom' => 1601856000
+ },
'timing' => {
'sslEn
+d' => -1,
'worke
+rReady' => -1,
'proxy
+End' => '0.359',
'conne
+ctStart' => -1,
'sendS
+tart' => '0.416',
'sslSt
+art' => -1,
'worke
+rRespondWithSettled' => -1,
'sendE
+nd' => '1.285',
'reque
+stTime' => '1492087.291689',
'proxy
+Start' => '0.097',
'conne
+ctEnd' => -1,
'worke
+rStart' => -1,
'pushS
+tart' => 0,
'dnsSt
+art' => -1,
'pushE
+nd' => 0,
'recei
+veHeadersEnd' => '49.727',
'worke
+rFetchStart' => -1,
'dnsEn
+d' => -1
}
},
'initiator' => {
'type' => 'other'
},
'request' => {
'initialPriority' => 'VeryHig
+h',
'headers' => {
'User-Agent' =
+> 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM
+L, like Gecko) HeadlessChrome/87.0.4280.88 Safari/537.36',
'Referer' => '
+https://tr.snapchat.com/cm/i?pid=a8067835-6f53-4ec6-b1d7-509ec7db92f4
+',
'Upgrade-Insec
+ure-Requests' => '1'
},
'referrerPolicy' => 'no-refer
+rer-when-downgrade',
'url' => 'https://tr.snapchat
+.com/cm/p?rand=1610201931907&pnid=140&pcid=f5cb7c21-5398-11eb-9bb4-ba
+2c2ef2346d',
'mixedContentType' => 'none',
'method' => 'GET'
},
'hasUserGesture' => $VAR1->[0]{'params'}{'re
+directResponse'}{'fromServiceWorker'},
'documentURL' => 'https://tr.snapchat.com/cm
+/p?rand=1610201931907&pnid=140&pcid=f5cb7c21-5398-11eb-9bb4-ba2c2ef23
+46d',
'frameId' => 'A04583AF0A872985D9548891D196F5
+E0'
}
},
{
'params' => {
'headers' => {
'cookie' => 'sc_at=v2|H4sIAAA
+AAAAAAE3GwRGAMAgEwIqYuQsnot0oCVWkeL/uazFWqLwM2YdptSwfvQbMriAdZ2wKN4Pw
+AV65f8UHQZX7ikAAAAA=',
'accept-language' => 'en-US',
'sec-fetch-mode' => 'navigate
+',
'upgrade-insecure-requests' =
+> '1',
':scheme' => 'https',
'referer' => 'https://tr.snap
+chat.com/cm/i?pid=a8067835-6f53-4ec6-b1d7-509ec7db92f4',
'user-agent' => 'Mozilla/5.0
+(Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)
+HeadlessChrome/87.0.4280.88 Safari/537.36',
'accept-encoding' => 'gzip, d
+eflate, br',
'accept' => 'text/html,applic
+ation/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apn
+g,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'sec-fetch-dest' => 'iframe',
':authority' => 'tr.snapchat.
+com',
':path' => '/cm/p?rand=161020
+1931907&pnid=140&pcid=f5cb7c21-5398-11eb-9bb4-ba2c2ef2346d',
':method' => 'GET',
'sec-fetch-site' => 'cross-si
+te'
},
'requestId' => 'EF16DAA4B81ECCB0A2A3FB76F0B0
+F3B6',
'frame' => {},
'associatedCookies' => [
{
'cookie' => {
'se
+cure' => $VAR1->[0]{'params'}{'redirectResponse'}{'connectionReused'}
+,
'pa
+th' => '/',
'ht
+tpOnly' => $VAR1->[0]{'params'}{'redirectResponse'}{'fromServiceWorke
+r'},
'si
+ze' => 104,
'na
+me' => 'sc_at',
'se
+ssion' => $VAR1->[0]{'params'}{'redirectResponse'}{'fromServiceWorker
+'},
'sa
+meSite' => 'None',
'ex
+pires' => '1644016199.48171',
'va
+lue' => 'v2|H4sIAAAAAAAAAE3GwRGAMAgEwIqYuQsnot0oCVWkeL/uazFWqLwM2Ydpt
+SwfvQbMriAdZ2wKN4PwAV65f8UHQZX7ikAAAAA=',
'do
+main' => '.snapchat.com',
'pr
+iority' => 'Medium'
},
'blockedReasons'
+=> []
}
]
},
'method' => 'Network.requestWillBeSentExtraInfo'
},
{
'params' => {
'frame' => {},
'blockedCookies' => [],
'headers' => {
'cf-request-id' => '07902953f
+a0000d28e22857000000001',
'x-shopify-generated-cart-tok
+en' => '6b9d64259b09cde535dac48f68879f6b',
'set-cookie' => 'secure_custo
+mer_sig=; path=/; expires=Mon, 10 Jan 2022 23:09:58 GMT; secure; Http
+Only
cart_currency=USD; path=/; expires=Sun, 24 Jan 2021 23:09:58 GMT; secu
+re; SameSite=None
cart_sig=32b470570732770b7820df9d419bfbd2; path=/; expires=Sun, 24 Jan
+ 2021 23:09:58 GMT; secure; HttpOnly; SameSite=None',
'cf-cache-status' => 'DYNAMIC
+',
'x-liquid-rendered-at' => '20
+21-01-10T23:09:58.861847156Z',
'cache-control' => 'no-cache,
+ no-store',
'x-shopid' => '2517377',
'x-xss-protection' => '1; mod
+e=block; report=/xss-report?source%5Baction%5D=app_liquid&source%5Bap
+p%5D=Shopify&source%5Bcontroller%5D=storefront_section%2Fapp_proxy&so
+urce%5Bsection%5D=storefront&source%5Buuid%5D=5a6f66a5-6c93-4bdb-9484
+-208b4ef7ecf4',
'server' => 'cloudflare',
'content-security-policy-repo
+rt-only' => '; report-uri /csp-report?source%5Baction%5D=app_liquid&s
+ource%5Bapp%5D=Shopify&source%5Bcontroller%5D=storefront_section%2Fap
+p_proxy&source%5Bsection%5D=storefront&source%5Buuid%5D=5a6f66a5-6c93
+-4bdb-9484-208b4ef7ecf4',
'x-sorting-hat-shopid' => '25
+17377',
'x-dc' => 'gcp-us-central1,gc
+p-us-central1,gcp-us-east1,gcp-us-east1',
'report-to' => '{"group":"net
+work-errors","max_age":2592000,"endpoints":[{"url":"https://monorail-
+edge.shopifycloud.com/v1/reports/nel/20190325/shopify"}]}
{"group":"network-errors","max_age":2592000,"endpoints":[{"url":"https
+://monorail-edge.shopifycloud.com/v1/reports/nel/20190325/shopify"}]}
+',
'x-download-options' => 'noop
+en',
'content-type' => 'applicatio
+n/json; charset=utf-8',
'cf-ray' => '60fa11999cb9d28e
+-DFW',
'content-language' => 'en',
'expect-ct' => 'max-age=60480
+0, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expec
+t-ct"',
'x-request-id' => '5a6f66a5-6
+c93-4bdb-9484-208b4ef7ecf4',
'content-encoding' => 'br',
'x-permitted-cross-domain-pol
+icies' => 'none',
'x-sorting-hat-podid' => '86'
+,
'x-content-type-options' => '
+nosniff',
'vary' => 'Accept-Encoding
Accept',
'content-security-policy' =>
+'; report-uri /csp-report?source%5Baction%5D=app_liquid&source%5Bapp%
+5D=Shopify&source%5Bcontroller%5D=storefront_section%2Fapp_proxy&sour
+ce%5Bsection%5D=storefront&source%5Buuid%5D=5a6f66a5-6c93-4bdb-9484-2
+08b4ef7ecf4',
'nel' => '{"report_to":"netwo
+rk-errors","max_age":2592000,"success_fraction":0.0001}
{"report_to":"network-errors","max_age":2592000,"success_fraction":0.0
+001}',
'x-shardid' => '86',
'x-shopify-stage' => 'product
+ion',
'strict-transport-security' =
+> 'max-age=7889238',
'date' => 'Sun, 10 Jan 2021 2
+3:09:58 GMT'
},
'requestId' => '21392.213'
},
'method' => 'Network.responseReceivedExtraInfo'
},
{
'params' => {
'frame' => {},
'headersText' => 'HTTP/1.1 200 OK
Access-Control-Allow-Credentials: true
Access-Control-Allow-Headers:
Access-Control-Allow-Methods: GET
Access-Control-Allow-Origin: *
Access-Control-Max-Age: 86400
Allow: OPTIONS, GET
Cache-Control: max-age=7200
Content-Type: application/json; charset=utf-8
Date: Sun, 10 Jan 2021 23:09:58 GMT
Server: nginx
Vary: Cookie
Content-Length: 46
Connection: keep-alive
',
'blockedCookies' => [],
'headers' => {
'Access-Control-Allow-Origin'
+ => '*',
'Content-Type' => 'applicatio
+n/json; charset=utf-8',
'Access-Control-Max-Age' => '
+86400',
'Vary' => 'Cookie',
'Date' => 'Sun, 10 Jan 2021 2
+3:09:58 GMT',
'Cache-Control' => 'max-age=7
+200',
'Connection' => 'keep-alive',
'Server' => 'nginx',
'Allow' => 'OPTIONS, GET',
'Content-Length' => '46',
'Access-Control-Allow-Credent
+ials' => 'true',
'Access-Control-Allow-Methods
+' => 'GET',
'Access-Control-Allow-Headers
+' => ''
},
'requestId' => '21392.241'
},
'method' => 'Network.responseReceivedExtraInfo'
},
{
'method' => 'DOM.attributeModified',
'params' => {
'frame' => {},
'value' => 'product-single__add-to-cart',
'nodeId' => 2990,
'name' => 'class'
}
},
{
'method' => 'DOM.childNodeCountUpdated',
'params' => {
'nodeId' => 2999,
'childNodeCount' => 1,
'frame' => {}
}
},
{
'method' => 'DOM.attributeModified',
'params' => {
'frame' => {},
'value' => 'false',
'name' => 'data-available',
'nodeId' => 2325
}
},
{
'params' => {
'previousNodeId' => 2307,
'parentNodeId' => 2275,
'node' => {
'attributes' => [],
'childNodeCount' => 0,
'backendNodeId' => 855,
'nodeType' => 1,
'localName' => 'div',
'nodeId' => 3011,
'nodeValue' => '',
'nodeName' => 'DIV'
},
'frame' => {}
},
'method' => 'DOM.childNodeInserted'
},
{
'method' => 'DOM.childNodeRemoved',
'params' => {
'nodeId' => 3011,
'parentNodeId' => 2275,
'frame' => {}
}
},
{
'params' => {
'url' => 'https://www.tenthousand.cc/product
+s/interval-short-no-liner?variant=32441933168727',
'frameId' => 'F7C8298C7F980A1FF5BAA43B19990B
+0D'
},
'method' => 'Page.navigatedWithinDocument'
}
];
Chrome behaviour problem: Didn't see a 'Network.responseReceived' even
+t for frameId A04583AF0A872985D9548891D196F5E0, requestId EF16DAA4B81
+ECCB0A2A3FB76F0B0F3B6, cannot synthesize response. I saw Network.requ
+estWillBeSent,Network.requestWillBeSentExtraInfo at /usr/local/share/
+perl5/site_perl/5.30/WWW/Mechanize/Chrome.pm line 2596.
