I am trying to extract product data (name, price, url) as well as which sizes and colours are available from the json formatted website (https://www.bergzeit.de/marken/salewa/). I cannot find a way to get to the 'elementList' where i can see the data encoded. I used this code to get to the correct script on the page.
scripts = response.xpath('//script/text()').getall()
the_script = [s for s in scripts if 'elementsList' in s]
data = json.loads(the_script[0])
I get JSONDecodeError: Expecting value: line 2 column 1 (char 1)
This is the HTML
<script>
window.__initialAppState = {
global: {
gtmId: 'GTM-NKB4VN4',
cookiePresent: false,
user: {
loggedIn: false,
b2b: false,
b2bPrefs: {},
b2bAdmin: false
},
modules: {
checkout: {
removePromoCodeUrl: '/summary/removevoucher/'
},
accountPage: {
},
loyaltyPage: {
urls: {
register: '/login/?registerAccountUrl=/register/loyalty/',
overview: '/club'
},
},
productsListPage: {
headline: JSON.parse("\"Salewa\""),
resultCount: 1754,
elementsList: [{"data":{"brand":{"name":"Salewa"},"description":"Wetterfest und ultrawarm für anspruchsvolle Herren: 550 cuin Daunenjacke mit wind- und wasserabweisender Außenschicht","features":[],"googleClick":{"event":"eec.productClick","ecommerce":{"click":{"actionField":{"list":"brand"},"products":[{"brand":"Salewa","category":"Marken/Salewa","id":"1080071-051","name":"Herren Brenta Jacke","position":"1","price":"167.18","variant":"dark olive|S"}]}}},"googleImpression":{"event":"eec.productImpression","ecommerce":{"impressions":[{"brand":"Salewa","category":"Marken/Salewa","id":"1080071-051","list":"brand","name":"Herren Brenta Jacke","position":"1","price":"167.18","variant":"dark olive|S"}]}},"id":"1080071","images":[{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic1.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic3.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic4.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic5.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic6.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic7.jpg","title":"Herren Brenta Jacke"}],"isNew":true,"itemId":"1080071-051","labels":["sale"],"manyAvailableSizes":true,"name":"Herren Brenta Jacke","price":{"savedAmount":"51,00 €","savedPercentage":20,"priceForSchemaOrgOffer":"198.95","previous":"249,95 €","old":"249,95 €","current":"198,95 €","conf":"189,00 €","isFromPrice":false,"isOldPrice":true,"isRetailPrice":false,"isSignificantlyReduced":true,"bestSavedAmount":"51,00 €"},"productId":"1080071","ratings":4.571428571428571,"ratingsCount":14,"sellingPoints":[],"sizedImages":[{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic1.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic1.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic3.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic3.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic4.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic4.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic5.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic5.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic6.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic6.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic7.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic7.jpg","title":"Herren Brenta Jacke"}}],"sustainable":true,"url":"/p/salewa-herren-brenta-jacke/1080071/#itemId=1080071-051","variations":[{"attributeName":"import:style","colorVariation":true,"display_string":"Herstellerfarbe","name":"var[import:style]","options":[{"isAnyAvailable":true,"isAvailable":false,"label":"autumnal","stock":0,"value":"4171-autumnal","valueThumb":"orange"},{"isAnyAvailable":true,"isAvailable":false,"label":"black out","stock":0,"value":"0910-black_out","valueThumb":"schwarz"},{"isAnyAvailable":true,"isAvailable":false,"label":"dark olive","stock":0,"value":"5281-dark_olive","valueThumb":"oliv"},{"isAnyAvailable":true,"isAvailable":false,"label":"electric","stock":0,"value":"8621-electric","valueThumb":"blau"},{"isAnyAvailable":true,"isAvailable":false,"label":"flame","stock":0,"value":"1501-flame","valueThumb":"rot"},{"isAnyAvailable":true,"isAvailable":false,"label":"gold","stock":0,"value":"2191-gold","valueThumb":"gelb"},{"isAnyAvailable":true,"isAvailable":false,"label":"syrah","stock":0,"value":"1571-syrah","valueThumb":"rot"},{"isAnyAvailable":true,"isAvailable":false,"label":"thyme","stock":0,"value":"5561-thyme","valueThumb":"gruen"}],"selectedOption":"5281-dark_olive"},{"attributeName":"import:manufacturer_size","colorVariation":false,"display_string":"Herstellergröße","name":"var[import:manufacturer_size]","options":[{"isAnyAvailable":true,"isAvailable":false,"label":"S","sameDiscount":true,"value":"S"},{"isAnyAvailable":true,"isAvailable":false,"label":"M","sameDiscount":true,"value":"M"},{"isAnyAvailable":true,"isAvailable":false,"label":"L","sameDiscount":true,"value":"L"},{"isAnyAvailable":true,"isAvailable":false,"label":"XL","sameDiscount":true,"value":"XL"},{"isAnyAvailable":true,"isAvailable":false,"label":"XXL","sameDiscount":true,"value":"XXL"}],"selectedOption":"S"}]},"type":"product"}],
oneTileTeaserInsertions: [],
salesTeaserInsertion: "",
highlightedArticlesIds: [],
selectableVariations: ["import:Var_Article"],
currentCategory: {"id":"-s-2831-salewa","name":"Salewa","path":"marken-salewa","url":"/marken/salewa/?show=all","root":false},
parentCategory: {"id":"-s-2831","name":"Marken","path":"marken","url":"/marken/?show=all","root":false},
subCategoryTeasers: [],
search: JSON.parse("null"),
ms: null,
isBot: false,
},
productPage: {
productData: {},
},
cartPage:
{}
,
wishlistPage: {
}
}
};
</script>
You're getting an error because the window.__initialAppState is not a JSON but a Javascript structure.
One way to do that is to treat this as a string and extract the elementsList, which can be parsed to a JSON. You can use a Regular Expression such as elementsList: (.*) for that. Then, you can handle the resulting string as a JSON using the python json module.
Hope this helps :)