youtube api - waiting for video

Question

I made a bot that waits for videos from several channels and sends new ones to telegram.. Here is the code:

from distutils.log import error
from googleapiclient.discovery import build
import config
import time

import telebot
ChannelsAndVideos = []
Channels = config.channels_id
bot_token = config.bot_token
api_key = config.api_key
sleepTime = config.sleepTime
messageText = config.messageText
telegram_channels_id = config.telegram_channels_id
youtube = build('youtube', 'v3', developerKey=api_key)
bot = telebot.TeleBot(bot_token)
sended = []
for ChannelId in Channels:
    try:
        request = youtube.channels().list(
            part='statistics',
            id=ChannelId
            )
        response = request.execute()
        if(response['pageInfo']['totalResults'] != 1):
            print("Error! Не удалось добавить " + ChannelId)
        else:
            print(ChannelId + " был добавлен (" + response.get('items')[0].get('statistics').get('videoCount') + " видео)")
            ChannelsAndVideos.append((ChannelId, response.get('items')[0].get('statistics').get('videoCount')))
    except error:
        print(error.message)
while(True):
    time.sleep(sleepTime)
    for i in range(len(ChannelsAndVideos)):
        request = youtube.channels().list(
            part='statistics',
            id=ChannelsAndVideos[i][0]
        )
        response = request.execute()
        if(response['pageInfo']['totalResults'] != 1):
            print("Error!")
        else:
            if response.get('items')[0].get('statistics').get('videoCount') > ChannelsAndVideos[i][1]:
                ChannelsAndVideos[i] = (ChannelsAndVideos[i][0], response.get('items')[0].get('statistics').get('videoCount'))
                request = youtube.channels().list(
                    part='contentDetails',
                    id=ChannelsAndVideos[i][0]
                    )
                response = request.execute()
                request = youtube.playlistItems().list(
                part=['contentDetails','snippet','status'],
                playlistId=response['items'][0]['contentDetails']['relatedPlaylists']['uploads']
                )
                response = request.execute()
                if not (response['items'][0]['snippet']['resourceId']['videoId'] in sended):
                    sended.append(response['items'][0]['snippet']['resourceId']['videoId'])
                    for chat_id in telegram_channels_id:
                       try:
                            bot.send_message(chat_id, messageText + "https://www.youtube.com/watch?v=" + response['items'][0]['snippet']['resourceId']['videoId'])
                       except ...:
                           print("Не удалось отправить сообщение в " + str(chat_id))

There is a problem in its implementation: every 30 seconds it makes a request for the latest videos of each channel, so this eats up RAM and the rest of possible requests (their number is limited in the youtube api). How can I optimize this system?

Answer 1

Reducing memory usage

 sended = []
 ...
                if not (response['items'][0]['snippet']['resourceId']['videoId'] in sended):
                    sended.append(response['items'][0]['snippet']['resourceId']['videoId'])

To avoid the ever growing list of sent videos, use a set. It will also improve the runtime complexity of ['videoId'] in sended from O(n) to O(1).

sended = set()
...
                if not (response['items'][0]['snippet']['resourceId']['videoId'] in sended):
                    sended.add(response['items'][0]['snippet']['resourceId']['videoId'])

Rate limiting

while(True):
    time.sleep(sleepTime)
    for i in range(len(ChannelsAndVideos)):
        request = youtube.channels().list(

Instead of sleeping in big chunk before iterating the channels, sleep before each request to the api. For easy usage, create a helper request function which will ensure at least sleepTime delay before the next request.

last_request_time = time.time()
def execute(request)
    global last_request_time
    next_request_time = last_request_time + sleepTime
    time.sleep(max(next_request_time - time.time(), 0))
    last_request_time = time.time()
    return request.execute()

Now replace all request.execute()s with execute(request). Adjust sleepTime value based on YouTube api usage limits and remove all time.sleep(sleepTime).

Result

from distutils.log import error
from googleapiclient.discovery import build
import config
import time

import telebot
ChannelsAndVideos = []
Channels = config.channels_id
bot_token = config.bot_token
api_key = config.api_key
sleepTime = config.sleepTime
messageText = config.messageText
telegram_channels_id = config.telegram_channels_id
youtube = build('youtube', 'v3', developerKey=api_key)
bot = telebot.TeleBot(bot_token)

sended = set()


last_request_time = 0
def execute(request)
    global last_request_time
    next_request_time = last_request_time + sleepTime
    time.sleep(max(next_request_time - time.time(), 0))
    last_request_time = time.time()
    return request.execute()


for ChannelId in Channels:
    try:
        request = youtube.channels().list(
            part='statistics',
            id=ChannelId
            )
        response = execute(request)
        if(response['pageInfo']['totalResults'] != 1):
            print(‎"Error! Failed to add "‎ + ChannelId)
        else:
            print(ChannelId + ‎" has been added ("‎ + response.get('items')[0].get('statistics').get('videoCount') + ‎" video)"‎)
            ChannelsAndVideos.append((ChannelId, response.get('items')[0].get('statistics').get('videoCount')))
    except error:
        print(error.message)

while(True):
    for i in range(len(ChannelsAndVideos)):
        request = youtube.channels().list(
            part='statistics',
            id=ChannelsAndVideos[i][0]
        )
        response = execute(request)
        if(response['pageInfo']['totalResults'] != 1):
            print("Error!")
        else:
            if response.get('items')[0].get('statistics').get('videoCount') > ChannelsAndVideos[i][1]:
                ChannelsAndVideos[i] = (ChannelsAndVideos[i][0], response.get('items')[0].get('statistics').get('videoCount'))
                request = youtube.channels().list(
                    part='contentDetails',
                    id=ChannelsAndVideos[i][0]
                    )
                response = execute(request)
                request = youtube.playlistItems().list(
                part=['contentDetails','snippet','status'],
                playlistId=response['items'][0]['contentDetails']['relatedPlaylists']['uploads']
                )
                response = execute(request)
                if not (response['items'][0]['snippet']['resourceId']['videoId'] in sended):
                    sended.add(response['items'][0]['snippet']['resourceId']['videoId'])
                    for chat_id in telegram_channels_id:
                       try:
                            bot.send_message(chat_id, messageText + "https://www.youtube.com/watch?v=" + response['items'][0]['snippet']['resourceId']['videoId'])
                       except ...:
                           print(‎"Failed to send message to "‎ + str(chat_id))