I get None when I am trying to fetch the videos from a Youtube channel. Attached is my code:
url = "https://www.youtube.com/@ecbeuro/videos"
service = Service("/usr/bin/chromedriver")
options = Options()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument('--disable-blink-features=AutomationControlled')
driver = webdriver.Chrome(service=service, options=options)
driver.get(url)
news_links = driver.find_elements(By.XPATH, '//*[@id="video-title"]')
for link in news_links:
print(link.get_attribute('href'))
Help is appreciated. Kind regards!
If your intention is to extract some of the latest videos from a YouTube channel, you can check this out to scrape 30 latest videos using just Python's requests library.
But if you would like to scrape all the available videos on a YouTube channel, you'll need to do multiple scrolling to load more/all the available videos. To achieve this, you can use Selenium.
import time
from selenium.webdriver import Chrome, ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
options = ChromeOptions()
options.add_argument("--start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
driver = Chrome(options=options)
wait = WebDriverWait(driver, 10)
driver.get("https://www.youtube.com/@ecbeuro/videos")
last_height = 0
print("Start scrolling!")
while True:
driver.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);")
new_height = driver.execute_script("return document.documentElement.scrollHeight")
wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div#dismissible')))
if last_height == new_height:
print("Stop scrolling, reached the bottom!")
break
else:
last_height = new_height
time.sleep(1)
data = []
videos = driver.find_elements(By.CSS_SELECTOR, 'div#dismissible')
for video in videos:
url = video.find_element(By.CSS_SELECTOR, 'div#thumbnail>ytd-thumbnail>a').get_attribute('href')
details = video.find_element(By.CSS_SELECTOR, 'div#details')
title = details.find_element(By.CSS_SELECTOR, 'div#meta>h3').text
views_date = details.find_elements(By.CSS_SELECTOR, 'span.inline-metadata-item.style-scope.ytd-video-meta-block')
views = views_date[0].text.strip()
date = views_date[1].text.strip()
data.append({"title": title, "url": url, "views": views, "posted_date": date})
print(f"Total videos: {len(data)}")
print(data)
output:
Start scrolling!
Stop scrolling, reached the bottom!
Total videos: 1480
[{'title': 'President Lagarde presents the latest monetary policy decisions – 27July 2023', 'url': 'https://www.youtube.com/watch?v=eUlRXBy3pBU', 'views': '2.3K views', 'posted_date': '5 days ago'}, {'title': 'Panel Discussion at the CESEE Conference 2023', 'url': 'https://www.youtube.com/watch?v=YMir_50lWhc', 'views': '746 views', 'posted_date': '13 days ago'}, {'title': 'Panel Discussion number 2 and Closing remarks at the CESEE Conference 2023', 'url': 'https://www.youtube.com/watch?v=bt-_Scd3864', 'views': '432 views', 'posted_date': '13 days ago'}, {'title': 'Civil Society Seminar Series: The evolution of European banking supervision', 'url': 'https://www.youtube.com/watch?v=d6MfWiHfua8', 'views': '412 views', 'posted_date': '2 weeks ago'}, {'title': 'Keynote speech of Valdis Dombrovskis at the CESEE Conference 2023', 'url': 'https://www.youtube.com/watch?v=-GnWOVKxFEk', 'views': '209 views', 'posted_date': '2 weeks ago'}, {'title': "Christine Lagarde's opening remarks for the CESEE Conference 2023", 'url': 'https://www.youtube.com/watch?v=pQzcvSXlI0M', 'views': '1K views', 'posted_date': '2 weeks ago'}, {'title': 'Keynote speech of Beata Javorcik at the CESEE Conference 2023', 'url': 'https://www.youtube.com/watch?v=rZITdIZYxBQ', 'views': '397 views', 'posted_date': '2 weeks ago'}, {'title': 'Civil Society Seminar Series: A digital euro for everyone', 'url': 'https://www.youtube.com/watch?v=gXD_7BDIn8Q', 'views': '1.8K views', 'posted_date': '2 weeks ago'}, {'title': 'New Euro banknotes re-design survey', 'url': 'https://www.youtube.com/watch?v=-ynWm1sYA9Q', 'views': '39K views', 'posted_date': '3 weeks ago'}, ....... {'title': 'ECB Press Conference - 13 January 2011 - Part 1/2', 'url': 'https://www.youtube.com/watch?v=fl_zhb4lW6c', 'views': '434 views', 'posted_date': '12 years ago'}, {'title': 'Preisstabilität: Warum ist sie für dich wichtig?', 'url': 'https://www.youtube.com/watch?v=6bSdXmxFcEE', 'views': '76K views', 'posted_date': '12 years ago'}, {'title': 'A estabilidade de preços é importante porquê?', 'url': 'https://www.youtube.com/watch?v=v4Zmx5OsKM8', 'views': '16K views', 'posted_date': '12 years ago'}, {'title': 'La stabilité des prix : pourquoi est-elle importante pour vous ?', 'url': 'https://www.youtube.com/watch?v=0xqcKYG9ax4', 'views': '37K views', 'posted_date': '12 years ago'}, {'title': 'Price stability: why is it important for you ?', 'url': 'https://www.youtube.com/watch?v=F6PvX625JCs', 'views': '66K views', 'posted_date': '12 years ago'}, {'title': 'Hinnastabiilsus – miks see on oluline?', 'url': 'https://www.youtube.com/watch?v=LhdGJ_g8k2M', 'views': '2.5K views', 'posted_date': '12 years ago'}, {'title': 'ECB Press Conference - 2 December 2010 - Part 1/2', 'url': 'https://www.youtube.com/watch?v=KsHgS6VslIk', 'views': '263 views', 'posted_date': '12 years ago'}, {'title': 'ECB Press Conference - 2 December 2010 - Part 2/2', 'url': 'https://www.youtube.com/watch?v=SP8PCanl93o', 'views': '221 views', 'posted_date': '12 years ago'}, {'title': 'ECB Statistics', 'url': 'https://www.youtube.com/watch?v=FyHiyPYyDp0', 'views': '3.3K views', 'posted_date': '12 years ago'}, {'title': 'The ECB launches new educational games', 'url': 'https://www.youtube.com/watch?v=HMIsUkNWKnE', 'views': '1.7K views', 'posted_date': '12 years ago'}, {'title': 'ECB - Inflation Island and Economia: Educational Games', 'url': 'https://www.youtube.com/watch?v=hcQUJSz82oQ', 'views': '9.5K views', 'posted_date': '12 years ago'}]