How to fetch multiple channel histories concurrently?

Question

So I'm trying to make a command for my Discord bot, where it will check every channel in a server and check the last message in each channel, and then send all the channels that start with the key variable.

  async def starthistory(self, ctx, key, msg, num):
      for channel in ctx.guild.text_channels:
          async for message in channel.history(limit=1):
              message_content = message.content.lower()
              if len(message.embeds) > 0:
                  if len(message.embeds[0].title) > 0:
                      message_content = message.embeds[0].title.lower()
                  elif len(message.embeds[0].author) > 0:
                      message_content = message.embeds[0].author.lower()
                  elif len(message.embeds[0].description) > 0:
                      message_content = message.embeds[0].description.lower()
                  
              if message_content.startswith(key.lower()):
                  num += 1
                  msg += f"\n**{num}.** {channel.mention} - **{channel.name}**"
                  
  #startswith
  @_list.command(name="starts_with",
                 aliases=["startswith", "sw", "s"],
                 brief="Lists all channels with message starting with <key>.",
                 help="Lists all channels with last message starting with the word/phrase <key>.",
                 case_insensitive=True)
  async def _starts_with(self, ctx, *, key):
      
      msg = f"Channels with last message starting with `{key}`:"
      num = 0
      wait = await ctx.send(f"Looking for messages starting with `{key}`...")

      asyncio.create_task(self.starthistory(ctx=ctx, key=key, msg=msg, num=num))
                
      if num == 0:
          msg += "\n**None**"
      msg += f"\n\nTotal number of channels = **{num}**"
      for para in textwrap.wrap(msg, 2000, expand_tabs=False, replace_whitespace=False, fix_sentence_endings=False, break_long_words=False, drop_whitespace=False, break_on_hyphens=False, max_lines=None):
          await ctx.send(para)
          await asyncio.sleep(0.5)
      await wait.edit(content="✅ Done.")

I want it to concurrently look at each channel's history so it doesn't take as long. Currently, my code doesn't change the already defined variables: num is always 0 and msg is always None.

How to concurrently look at each channel's history instead of one at a time?

Answer 1

asyncio.create_task(coro) creates an asynchronous task and runs it in the background. To allow your for loop to run asynchronously, where all the text channels are being processed at the same time, you should use asyncio.gather(coros) instead.

Here is the working code (I trimmed down your code to only the relevant parts):

@staticmethod
async def check_history(msgs, channel, key, semaphore):
    async with semaphore:
        async for message in channel.history(limit=1):
            message_content = message.content.lower()
            # trimmed some code...
            if message_content.startswith(key.lower()):
                num = len(msgs)
                msgs += [f"**{num}.** {channel.mention} - **{channel.name}**"]


@_list.command()
async def _starts_with(self, ctx, *, key):
    msgs = [f"Channels with last message starting with `{key}`:"]
    tasks = []
    semaphore = asyncio.Semaphore(10)

    for channel in ctx.guild.text_channels:
        tasks += [self.check_history(msgs, channel, key, semaphore)]

    await asyncio.gather(*tasks)

    if len(msgs) == 1:
        msgs += ["**None**"]
    msgs += [f"\nTotal number of channels = **{len(msgs)}**"]
    msg = "\n".join(msgs)
    print(msg)

Main points of note/why this works:

• I used asyncio.gather() to await all of the check_history coroutines.
• I used a asyncio.Semaphore(10) which will restrict the max concurrency to 10. Discord API doesn't like it when you send too many requests at the same time. With too much channels, you might get temporary blocked.
• Usually, it's not recommended pass immutable objects (strs and ints) to an external function and attempt to change its values. For your use case, I believe the best alternative is to use a msg list then str.join() it later. This gets rid of num as well.