C# Sharepoint.Client - Return all files and folders from a given subfolder

Question

I am trying to return all files and folders in a SharePoint library starting from a given subfolder.

If I set the FolderServerRelativeUrl on the CamlQuery to the folder i wish to start from, I can get all the list items for that given folder; however, when I try to add in camlQuery.ViewXML to recursively return items with any additional sub folders as well, I get the following exception:

Microsoft.SharePoint.Client.ServerException: 'The attempted operation is prohibited because it exceeds the list view threshold enforced by the administrator.'

Code

public static IEnumerable<string> GetSharepointFiles2(string sharePointsite, string libraryName, string username, string password, string subFolders)
{
    Uri filename = new Uri(sharePointsite);
    string server = filename.AbsoluteUri.Replace(filename.AbsolutePath, "");
    List<string> fullfilePaths = new List<string>();

    using (ClientContext cxt = new ClientContext(filename))
    {
        cxt.Credentials = GetCreds(username, password);

        Web web = cxt.Web;
        cxt.Load(web, wb => wb.ServerRelativeUrl);
        cxt.ExecuteQuery();

        List list = web.Lists.GetByTitle(libraryName);
        cxt.Load(list);
        cxt.ExecuteQuery();

        Folder folder = web.GetFolderByServerRelativeUrl(web.ServerRelativeUrl + subFolders);
        cxt.Load(folder);
        cxt.ExecuteQuery();

        CamlQuery camlQuery = new CamlQuery();
        camlQuery.ViewXml = @"<View Scope='RecursiveAll'>
                             <Query>
                             </Query>
                         </View>";

        camlQuery.FolderServerRelativeUrl = folder.ServerRelativeUrl;
        ListItemCollection listItems = list.GetItems(camlQuery);

        cxt.Load(listItems);
        cxt.ExecuteQuery();

        foreach (ListItem listItem in listItems)
        {
            if (listItem.FileSystemObjectType == FileSystemObjectType.File)
            {

                fullfilePaths.Add(String.Format("{0}{1}", server, listItem["FileRef"]));
            }
            else if (listItem.FileSystemObjectType == FileSystemObjectType.Folder)
            {
                Console.WriteLine(String.Format("{0}{1}", server, listItem["FileRef"]));
                fullfilePaths.Add(String.Format("{0}{1}", server, listItem["FileRef"]));
            }
        }
    }
    return fullfilePaths;
}

private static SharePointOnlineCredentials GetCreds(string username, string password)
{
    SecureString securePassword = new SecureString();

    foreach (char c in password.ToCharArray()) securePassword.AppendChar(c);
    return new SharePointOnlineCredentials(username, securePassword);
}

In terms of the threshold limit, I have tried this on a folder with only 1 file and 1 folder in (in turn that folder has 1 file only), so if the limit is default 5000, I have no idea why I'd be getting this.

Answer 1

Finally found a solution that works, even if it is a bit sledgehammer!

Although the folders I was looking to retrieve the items for had far less than 5000 items, the issue was the list as a whole did exceed this threshold (In this case it was about 11,000 items).

I removed the FolderServerRelativeURL attribute and then used ListItemCollectionPosition to paginate/batch up the all the items in the list. Once all the items are in a collection it can be filtered with Linq for the relevant subfolder. (CAML Query - Going around the 5000 List Item Threshold)

If anyone has a way of being more targeted with the items, I'd love to see it.

Code:

public static IEnumerable<ListItem> GetSharepointFiles2(string sharePointsite, string libraryName, string username, string password, string subFolders)
{
    Uri filename = new Uri(sharePointsite);
    List<ListItem> items = new List<ListItem>();

    using (ClientContext cxt = new ClientContext(filename))
    {
        cxt.Credentials = GetCreds(username, password);

        Web web = cxt.Web;
        cxt.Load(web, wb => wb.ServerRelativeUrl);
        cxt.ExecuteQuery();

        List list = web.Lists.GetByTitle(libraryName);
        cxt.Load(list);
        cxt.ExecuteQuery();

        CamlQuery camlQuery = new CamlQuery();
        camlQuery.ViewXml = "<View Scope='Recursive'><RowLimit>5000</RowLimit></View>";

        do
        {
            ListItemCollection listItems = list.GetItems(camlQuery);
            cxt.Load(listItems);
            cxt.ExecuteQuery();

            items.AddRange(listItems);
            camlQuery.ListItemCollectionPosition = listItems.ListItemCollectionPosition;

        } while (camlQuery.ListItemCollectionPosition != null);

        var filteritems = items.Where(tt => tt.FieldValues["FileRef"].ToString().StartsWith(web.ServerRelativeUrl + subFolders));

        return filteritems;
    }
}