How to check if link is borken or closed with webrequest?

Question

I am getting data from news website's RSS and show in listview.(e.g Title, Description, pubDate, Tags, Images and Comment Count ) but I have little problem when link is broken or closed, I am taking error. (An unhandled exception of type 'System.AggregateException' occurred in mscorlib.dll ) I want to check if link is broken or closed webrequest skip this adress and continue to other adress.

First of all check topics about webrequest but I didnt find answer like I want. just this topic give a idea but I didnt integration to my code. (how to send web request to check the link whether the link is broken or not)

thank you for help now.

This is my main class

public static Dictionary<string, HaberYildizi> HaberYildiziHaberList = new Dictionary<string, HaberYildizi>();
    public static bool degismisMi = false;

    public Dictionary<string, HaberYildizi> GetTagsHaberYildizi()
    {
        List<IBaseClass> yeniHaberList = new List<IBaseClass>();
        degismisMi = false;

        XmlDocument xdoc = new XmlDocument();
        xdoc.Load("http://www.haberyildizi.com/rss.php");
        XmlElement el = (XmlElement)xdoc.SelectSingleNode("/rss");

        if (el != null)
            el.ParentNode.RemoveChild(el);

        XmlNode Haberler = el.SelectSingleNode("channel");

        List<HaberYildizi> newHaberYildizi = new List<HaberYildizi>();
        string htmlStr = String.Empty;

        foreach (XmlNode haber in Haberler.SelectNodes("item"))
        {
            var link = haber.SelectSingleNode("link").InnerText;

            if (HaberYildiziHaberList.ContainsKey(link))
                continue;

            HaberYildizi haberYildizi = new HaberYildizi();
            haberYildizi.Title = WebUtility.HtmlDecode(haber.SelectSingleNode("title").InnerText);

            if (haber.SelectSingleNode("description").InnerText.Contains("</a>"))
            {
                var str1 = haber.SelectSingleNode("description").InnerText.IndexOf("</a>");
                var str2 = haber.SelectSingleNode("description").InnerText.Substring(str1 + 4);
                haberYildizi.Description = WebUtility.HtmlDecode(str2);
            }
            else
            {
                haberYildizi.Description = WebUtility.HtmlDecode(haber.SelectSingleNode("description").InnerText);
            }

            haberYildizi.Image = haber.SelectSingleNode("image").InnerText;

            haberYildizi.Link = link;
            var format = DateTime.Parse(haber.SelectSingleNode("pubDate").InnerText.Replace("Z", ""));
            haberYildizi.PubDate = format;

        //*************************************
            htmlStr = Utilities.GetResponseStr(haberYildizi.Link).Result; // mistake is here 
        //*************************************
            haberYildizi.Tags = GetTags(htmlStr);

            haberYildizi.Comment = GetCommentCount(htmlStr);

            if (HaberYildiziHaberList.ContainsKey(haber.SelectSingleNode("link").InnerText) == false)
            {
                degismisMi = true;
                HaberYildiziHaberList.Add(link, haberYildizi);
                newHaberYildizi.Add(haberYildizi);
            }
            yeniHaberList.Add(haberYildizi);
        }
        return HaberYildiziHaberList;
    }
    public int GetCommentCount(string htmlStr)
    {
        int count = 0;
        string commentKeyword = "label label-important\">";
        var comment = htmlStr.IndexOf(commentKeyword) + 23;
        if (comment != -1)
        {
            var comment2 = htmlStr.IndexOf("</span>", comment);

            var comment4 = htmlStr.Substring(comment, comment2 - comment).Trim();
            count = Convert.ToInt32(comment4);

        }
        return count;
    }
    public List<string> GetTags(string htmlStr)
    {            
        List<string> listele = new List<string>();
        string begenningKeyword = "<meta name=\"keywords\" content=\"";

        var tags = htmlStr.IndexOf(begenningKeyword);

        var final_response2 = htmlStr.Substring(tags + begenningKeyword.Length);
        var tagsBol = final_response2.IndexOf("\" />");

        var lastTags = final_response2.Substring(0, tagsBol);
        var tagsSonuc = lastTags.Split(',');
        foreach (var tag in tagsSonuc)
        {
            listele.Add(tag);
        }
        return listele;
    }
}

This my Utilities class

public class Utilities
{
    public static async Task<string> GetResponseStr(string link)
    {
        HttpWebRequest request = (HttpWebRequest)WebRequest.Create(link);
        HttpWebResponse response = (HttpWebResponse)request.GetResponse();

        StreamReader stream = new StreamReader(response.GetResponseStream());
        string final_response = stream.ReadToEnd();
        return final_response;
    }

}

Answer 1

You can just simply integrate to your code like below:

public static async Task<string> GetResponseStr(string link)
        {
            var final_response = string.Empty;
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(link);
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                StreamReader stream = new StreamReader(response.GetResponseStream());
                final_response = stream.ReadToEnd();
            }
            catch (Exception ex)
            {
                //DO whatever necessary like log or sending email to notify you   
            }

            return final_response;
        }

Then when you call, add a check before processing any further:

    htmlStr = Utilities.GetResponseStr(haberYildizi.Link).Result;
    if (!string.IsNullOrEmpty(htmlStr))
    {

    }