I am new to c# and JSON. I am trying to parse a JSON URL which has a request limit of 100 objects per page. I am not able to retrieve pages beyond the first page with 100 objects even though there're 2022 objects in 21 pages.
What am I missing in my code to allow the parsing of all 22 pages of JSON objects?
Thank you for all of your help in advance!
The total count of objects is given by the HTTP response header: X-Total-Count
public class VCUtils
{
public const string HeaderRateLimit = "X-Rate-Limit-Limit";
public const string HeaderRateRemaining = "X-Rate-Limit-Remaining";
public const string HeaderRateReset = "X-Rate-Limit-Reset";
public const string HeaderCountTotal = "X-Total-Count";
}
The following is my testing code.
protected void Page_Load(object sender, EventArgs e)
{
string baseURL = "https://api.veracross.com/vcdemo3/v2/students.json?page=";
int recordcount = 1;
int nextPage = 1;
int totalObjectCount = 0;
int defaultObjectsPerPage = 100;
int pageCount = 1000;
try
{
while (nextPage <= pageCount) //pageCount
{
string jsonData = "";
var req = (HttpWebRequest)WebRequest.Create(baseURL + nextPage);
req.Credentials = new NetworkCredential(vcAPIUsername, vcAPIPassword);
req.PreAuthenticate = true;
HttpWebResponse response = req.GetResponse() as HttpWebResponse;
using (var reader = new System.IO.StreamReader(response.GetResponseStream())) { jsonData = reader.ReadToEnd(); }
// get total number of objects and calculate the page count
if (response.Headers[VCUtils.HeaderCountTotal] != null)
{
totalObjectCount = Convert.ToInt16(response.Headers[VCUtils.HeaderCountTotal]);
pageCount = (int)Math.Ceiling((double)totalObjectCount / (double)defaultObjectsPerPage);
}
VCStudents studentObj = new VCStudents();
IList<VCStudents> validStudents;
// Call the JSON.NET deserializer helper method
validStudents = VCUtils.DeserializeToList<VCStudents>(jsonData);
foreach (var item in validStudents)
{
var studs = item.GetType().GetProperties();
//**
int stkey = item.person_pk; //possibly legacy fields
string surname = item.last_name;
string first_name = item.first_name;
string pref_name = item.preferred_name;
string st_mobile = item.mobile_phone;
string st_email = item.email_1;
int gsis_id = item.person_pk;
string birthdate = item.birthday;
string gender = item.gender;
string lastupdatedate = item.update_date;
var fdkey = item.first_name;
Literal1.Text = Literal1.Text + recordcount + ") " + stkey + " " + surname + " " + first_name + " " + pref_name + " " + st_mobile + " " + st_email + " " + gsis_id + " " + birthdate + " " + roll_group + " " + gender + " " + fdkey + " " + lastupdatedate + "</br> ";
recordcount = recordcount + 1;
}
nextPage = nextPage + 1;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
finally
{
}
}
The following is the JsonHelper method referenced from, http://www.cshandler.com/2013/09/deserialize-list-of-json-objects-as.html#.WPguVUWGOJA
public static List<string> InvalidJsonElements;
public static IList<T> DeserializeToList<T>(string jsonString)
{
InvalidJsonElements = null;
var array = JArray.Parse(jsonString);
IList<T> objectsList = new List<T>();
foreach (var item in array)
{
try
{
// CorrectElements
objectsList.Add(item.ToObject<T>());
}
catch (Exception ex)
{
InvalidJsonElements = InvalidJsonElements ?? new List<string>();
InvalidJsonElements.Add(item.ToString());
}
}
return objectsList;
}
The following is the class generated on json2csharp.com
public class VCStudentRole
{
public string description { get; set; }
}
public class VCStudents
{
public const string studentsBasePath = "students";
public int household_fk { get; set; }
public int person_pk { get; set; }
public string name_prefix { get; set; }
public string first_name { get; set; }
public string first_nick_name { get; set; }
public string preferred_name { get; set; }
public string middle_name { get; set; }
public string last_name { get; set; }
public string name_suffix { get; set; }
public string birthday { get; set; }
public string gender { get; set; }
public string email_1 { get; set; }
public bool display_email_1 { get; set; }
public string email_2 { get; set; }
public bool display_email_2 { get; set; }
public string home_phone { get; set; }
public bool display_home_phone { get; set; }
public string mobile_phone { get; set; }
public bool display_mobile_phone { get; set; }
public int advisor_fk { get; set; }
public string advisor_name { get; set; }
public int homeroom_teacher_fk { get; set; }
public string homeroom_teacher_name { get; set; }
public int homeroom { get; set; }
public List<VCStudentRole> roles { get; set; }
public string current_grade { get; set; }
public string grade_applying_for { get; set; }
public int year_applying_for { get; set; }
public bool resident_status_applying_for { get; set; }
public bool student_group_applying_for { get; set; }
public bool campus_applying_for { get; set; }
public string school_level { get; set; }
public string enrollment_status { get; set; }
public bool new_student { get; set; }
public int graduation_year { get; set; }
public string resident_status { get; set; }
public string campus { get; set; }
public string dorm { get; set; }
public string room_number { get; set; }
public int floor_number { get; set; }
public int bed_number { get; set; }
public string mailbox_number { get; set; }
public string student_group { get; set; }
public int parent_1_fk { get; set; }
public int parent_2_fk { get; set; }
public int parent_3_fk { get; set; }
public int parent_4_fk { get; set; }
public string username { get; set; }
public string update_date { get; set; }
}
I doubt if you get the total number accurate or not with this code:
totalObjectCount = Convert.ToInt16(response.Headers[VCUtils.HeaderCountTotal]);
You can try this:
totalObjectCount = Convert.ToInt32(response.Headers.Get(VCUtils.HeaderCountTotal));
See more details from MSDN: https://msdn.microsoft.com/en-us/library/system.net.webheadercollection(v=vs.110).aspx