Mypy errors out on Calling function with return type TypeVar of List of TypedDict

Im attempting to type hint functions in a class but I'm getting the following error in mypy

The get_items function calls the scrapyRT API running in the background and outputs depending on the spider name and url the spider espn-matches returns a output in the form of List[Match] and espn-players returns a output in the form of List[Player]

The get_items function returns either a list of Player dict or a list of Matches dict

app/fantasy_cricket/ScrapyRTClient.py:74: error: Incompatible return value type (got "List[Player]", expected "List[Match]")
Found 1 error in 1 file (checked 1 source file)

This is my code:

import requests
import sys
if sys.version_info >= (3, 8):
    from typing import TypedDict
else:
    from typing_extensions import TypedDict
from typing import TypeVar, List, Union, Optional

class Player(TypedDict):
    name: str
    role: str
    image: str
    player_id: str

class Match(TypedDict):
    runs: Optional[int]
    boundaries: Optional[int] 
    sixes: Optional[int]
    wicket : Optional[int]
    Maiden: Optional[int]
    Catch: Optional[int]
    Stump: Optional[int]
    match_id: str

Item = TypeVar("Item", Player, Match)

class espn_scrapyrt_client:
    def __init__(self) -> None:

        self.url = "http://localhost:9080/crawl.json"

    def get_items(self, spider_name: str, url: str) -> List[Item]:

        items = requests.get(self.url, params={"spider_name": spider_name, "url": url})

        return items.json()["items"]

    def get_player_dets(self, team: str, players: List[str]) -> List[Player]:

        team_map = {
            "India": "https://www.espncricinfo.com/ci/content/player/index.html?country=6",
            "Australia": "https://www.espncricinfo.com/ci/content/player/index.html?country=2",
            "England": "https://www.espncricinfo.com/ci/content/player/index.html?country=1",
            "Bangladesh": "https://www.espncricinfo.com/ci/content/player/index.html?country=25",
            "New Zealand": "https://www.espncricinfo.com/ci/content/player/index.html?country=5",
            "South Africa": "https://www.espncricinfo.com/ci/content/player/index.html?country=3",
            "Pakistan": "https://www.espncricinfo.com/ci/content/player/index.html?country=7",
            "Sri Lanka": "https://www.espncricinfo.com/ci/content/player/index.html?country=8",
            "West Indies": "https://www.espncricinfo.com/ci/content/player/index.html?country=4",
            "Afghanistan": "https://www.espncricinfo.com/ci/content/player/index.html?country=40",
        }

        team_players = self.get_items(spider_name="espn-players", url=team_map[team])
        names = {player["name"]: i for i, player in enumerate(team_players)}
        player_det = []
        for player in players:
            player_name = player.strip().split()
            if len(player_name) == 1:
                p = [names[name] for name in names if player.strip() in name]
            elif len(player_name) > 1:
                p = [
                    names[name]
                    for name in names
                    if player_name[0] in name and player_name[-1] in name
                ]
            if p!=[]:
                player_det.append(team_players[p[0]])
        return player_det

    def get_match_de(self, player_id: str) -> List[Match]:

        x = self.get_items(spider_name="espn-matches", url = "https://stats.espncricinfo.com/ci/engine/player/253802.html?class=1&orderby=start&orderbyad=reverse&template=results&type=allround&view=match")
        print(x)
        return x //error here

Any help would be appreciated! Thanks!!

Solution

The issue here is in these two lines, in your get_items method:

items = requests.get(self.url, params={"spider_name": spider_name, "url": url})
return items.json()["items"]

MyPy has no clue what the types of the keys and values will be in the dict resulting from a call to items.json(). MyPy's confusion here then percolates throughout the rest of your code, and eventually creates an error in line 74.

We can fix this by using typing.cast to give MyPy some information about the types of the keys and values in this dictionary. We can also use typing.overload, in combination with typing.Literal, to inform MyPy that if get_items is called with spider_name="espn-players", it will return a list of Player dicts, whereas if it is called with spider_name="espn-matches", it will return a list of Match dicts:

import requests
import sys

if sys.version_info >= (3, 8):
    from typing import TypedDict
else:
    from typing_extensions import TypedDict
    
from typing import List, Union, Optional, overload, Literal, cast


class Player(TypedDict):
    name: str
    role: str
    image: str
    player_id: str


class Match(TypedDict):
    runs: Optional[int]
    boundaries: Optional[int] 
    sixes: Optional[int]
    wicket : Optional[int]
    Maiden: Optional[int]
    Catch: Optional[int]
    Stump: Optional[int]
    match_id: str
    

ListOfPlayersOrMatches = Union[List[Player], List[Match]]


class ResponseDict(TypedDict):
    items: ListOfPlayersOrMatches


class espn_scrapyrt_client:
    def __init__(self) -> None:
        self.url = "http://localhost:9080/crawl.json"
        
    @overload
    def get_items(self, spider_name: Literal["espn-players"], url: str) -> List[Player]:
        ...
        
    @overload
    def get_items(self, spider_name: Literal["espn-matches"], url: str) -> List[Match]:
        ...

    def get_items(
        self, 
        spider_name: Literal["espn-players", "espn-matches"], 
        url: str
    ) -> ListOfPlayersOrMatches:
        
        response = requests.get(self.url, params={"spider_name": spider_name, "url": url})
        response_dict = cast(ResponseDict, response.json())
        return response_dict["items"]

    def get_player_dets(self, team: str, players: List[str]) -> List[Player]:
        team_map = {
            "India": "https://www.espncricinfo.com/ci/content/player/index.html?country=6",
            "Australia": "https://www.espncricinfo.com/ci/content/player/index.html?country=2",
            "England": "https://www.espncricinfo.com/ci/content/player/index.html?country=1",
            "Bangladesh": "https://www.espncricinfo.com/ci/content/player/index.html?country=25",
            "New Zealand": "https://www.espncricinfo.com/ci/content/player/index.html?country=5",
            "South Africa": "https://www.espncricinfo.com/ci/content/player/index.html?country=3",
            "Pakistan": "https://www.espncricinfo.com/ci/content/player/index.html?country=7",
            "Sri Lanka": "https://www.espncricinfo.com/ci/content/player/index.html?country=8",
            "West Indies": "https://www.espncricinfo.com/ci/content/player/index.html?country=4",
            "Afghanistan": "https://www.espncricinfo.com/ci/content/player/index.html?country=40",
        }

        team_players = self.get_items(spider_name="espn-players", url=team_map[team])
        names = {player["name"]: i for i, player in enumerate(team_players)}
        player_det = []
        
        for player in players:
            player_name = player.strip().split()
            if len(player_name) == 1:
                p = [names[name] for name in names if player.strip() in name]
            elif len(player_name) > 1:
                p = [
                    names[name]
                    for name in names
                    if player_name[0] in name and player_name[-1] in name
                ]
            if p!=[]:
                player_det.append(team_players[p[0]])
        return player_det


    def get_match_de(self, player_id: str) -> List[Match]:
        x = self.get_items(spider_name="espn-matches", url = "https://stats.espncricinfo.com/ci/engine/player/253802.html?class=1&orderby=start&orderbyad=reverse&template=results&type=allround&view=match")
        print(x)
        return x