How to retrieve more than 100 messages from the history of a PubNub channel?

The page about the PubNub History API states that

The history() function returns a list of up to 100 messages, the start time token and the ending time token.

Is there a way to retrieve more than the 100 messages?

I'm currently not a paying customer of PubNub.

Solution

PubNub Load History More than 100 Messages

Sometimes you want to slice back in time over a linear stream of data. And often you'll want to do this at different levels of granularity. That is why PubNub Storage and Playback APIs provide maximum level of flexibility. However sometimes it ends up being a bit tricky to load data with the preferred result set.

PubNub Real-Time Network Storage and Playback

There are several considerations you may be seeking when loading transaction history over timelines that can potentially span millions of message in the transaction set. There are some great options available to you and we will cover two of them right now. The examples will be coded in JavaScript. The first example loads a summary of the data by grabbing the snapshots for the beginning of each hour for the past 24 hours. The second example shows you how to load all transactions in full detail and maximum granularity.

All Reference Files can be found on this GIST: Loading History from PubNub Mt.Gox Trades

Example PubNub Mt.Gox History JavaScript Usage

<script src="https://cdn.pubnub.com/pubnub.min.js"></script>
<script src="mtgox-history.js"></script>
<script>(function(){

// LOAD HOURLY SUMMARY
MTGOX.history.hourly({
    channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
    data    : function(response) { console.log(JSON.stringify(response)) },
    error   : function()         { console.log("NETWORK ERROR")  } 
});

// LOAD ALL WITH LIMITER OPTION
MTGOX.history.full({
    limit   : 500, // SET LIMIT AS HIGH AS NEEDED TO LOAD MORE!
    channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
    data    : function(messages) { console.log(messages)        },
    error   : function(e)        { console.log("NETWORK ERROR") }
});

})();</script>

NOTE: Running MTGOX.history.hourly() method will generate a list of snapshots per hour over the last 24 hours.

NOTE: Running MTGOX.history.full() method will generate maximum resolution detail with a lot of data. You can get a full dump or partial dump as needed; and you should increase the limit parameter in order to grab more data points.

This following JavaScript file will provide you the MTGOX interface.

PubNub Mt.Gox History JavaScript Loader

//
// mtgox-history.js
//

(function(){

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// INITIALIZE PUBNUB
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
var pubnub = PUBNUB.init({
    subscribe_key : 'sub-c-50d56e1e-2fd9-11e3-a041-02ee2ddab7fe'
});

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// MTGOX HISTORY INTERFACE
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
window.MTGOX = {
    history : {
        hourly : hourly,
        full   : full
    }
};

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// GET ALL DATA FOREVER (WITH LIMIT OF COURSE)
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

/*
MTGOX.history.full({
    limit   : 1000,
    channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
    data    : function(messages) { console.log(messages)        },
    error   : function(e)        { console.log("NETWORK ERROR") }
});
*/

function full(args) {
    var chan     = args['channel'] ||'d5f06780-30a8-4a48-a2f8-7ed181b4a13f'
    ,   callback = args['data']   || function(){}
    ,   error    = args['error']  || function(){}
    ,   limit    = +args['limit'] || 5000
    ,   start    = 0
    ,   count    = 100
    ,   history  = []
    ,   params   = {
            channel  : chan,
            count    : count,
            callback : function(messages) {
                var msgs = messages[0];
                start = messages[1];
                params.start = start;
                PUBNUB.each( msgs.reverse(), function(m) {history.push(m)} );

                if (history.length >= limit) return callback(history);
                if (msgs.length < count)     return callback(history);

                count = 100;
                add_messages();
            },
            error : function(e) {
                callback(history);
                error(history);
            }
        };

    add_messages();
    function add_messages() { pubnub.history(params) }
}

// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
// GET 24 HOURS IN HOURLY INCREMENTS
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

/*
MTGOX.history.hourly({
    channel : 'd5f06780-30a8-4a48-a2f8-7ed181b4a13f',
    data    : function(response) { console.log(response) },
    error   : function()         { console.log('ERROR')  } 
});
*/

function hourly(setup) {
    var limit = 24;
    var count = 0;
    var chan  = setup['channel'] ||'d5f06780-30a8-4a48-a2f8-7ed181b4a13f';
    var cb    = setup['data']    || function(){};
    var eb    = setup['error']   || function(){};
    var now   = new Date();

    now.setUTCHours(0);
    now.setUTCMinutes(0);
    now.setUTCSeconds(0);
    now.setUTCMilliseconds(0);

    var utc_now = now.getTime();
    var vectors = [];

    PUBNUB.each( (new Array(limit)).join(',').split(','), function( _, d ) {
        var day = utc_now - 3600000 * d;
        pubnub.history({
            limit    : 1,
            channel  : chan,
            start    : day * 10000,
            error    : function() { count++; eb(); },
            callback : function(messages) {
                // DONE?
                if (++count == limit) return cb(vectors);

                // ADD TIME SLICES
                var res = +(((messages[0][0]||{}).ticker||{}).avg||{}).value;
                res && vectors.push([ new Date(day).getUTCHours(), res ]);

                // KEEP IT SORTED
                vectors.sort(function(a,b){ return a[0] > b[0] && -1 || 1 });
            }
        })
    } );
}

})();

Mt.Gox PubNub Channel Listing for Tickers, Depth and Trades

The following is a list of channels provided by Mt.Gox data feed options you can use in the history channel parameter field.

{
"TICKER.ltcgbp": "0102A446-E4D4-4082-8E83-CC02822F9172",
"TICKER.ltccny": "0290378C-E3D7-4836-8CB1-2BFAE20CC492",
"DEPTH.btchkd": "049F65DC-3AF3-4FFD-85A5-AAC102B2A579",
"DEPTH.btceur": "057BDC6B-9F9C-44E4-BC1A-363E4443CE87",
"TICKER.nmcaud": "08C65460-CBD9-492E-8473-8507DFA66AE6",
"TICKER.btceur": "0BB6DA8B-F6C6-4ECF-8F0D-A544AD948C15",
"DEPTH.btckrw": "0C84BDA7-E613-4B19-AE2A-6D26412C9F70",
"DEPTH.btccny": "0D1ECAD8-E20F-459E-8BED-0BDCF927820F",
"TICKER.btccad": "10720792-084D-45BA-92E3-CF44D9477775",
"DEPTH.btcchf": "113FEC5F-294D-4929-86EB-8CA4C3FD1BED",
"TICKER.ltcnok": "13616AE8-9268-4A43-BDF7-6B8D1AC814A2",
"TICKER.ltcusd": "1366A9F3-92EB-4C6C-9CCC-492A959ECA94",
"TICKER.btcbtc": "13EDFF67-CFA0-4D99-AA76-52BD15D6A058",
"TICKER.ltccad": "18B55737-3F5C-4583-AF63-6EB3951EAD72",
"TICKER.nmccny": "249FDEFD-C6EB-4802-9F54-064BC83908AA",
"DEPTH.btcusd": "24E67E0D-1CAD-4CC0-9E7A-F8523EF460FE",
"TICKER.btcchf": "2644C164-3DB7-4475-8B45-C7042EFE3413",
"DEPTH.btcaud": "296EE352-DD5D-46F3-9BEA-5E39DEDE2005",
"TICKER.btcczk": "2A968B7F-6638-40BA-95E7-7284B3196D52",
"TICKER.btcsgd": "2CB73ED1-07F4-45E0-8918-BCBFDA658912",
"TICKER.nmcjpy": "314E2B7A-A9FA-4249-BC46-B7F662ECBC3A",
"TICKER.btcnmc": "36189B8C-CFFA-40D2-B205-FB71420387AE",
"DEPTH.btcinr": "414FDB18-8F70-471C-A9DF-B3C2740727EA",
"DEPTH.btcsgd": "41E5C243-3D44-4FAD-B690-F39E1DBB86A8",
"TICKER.btcltc": "48B6886F-49C0-4614-B647-BA5369B449A9",
"TICKER.ltceur": "491BC9BB-7CD8-4719-A9E8-16DAD802FFAC",
"TICKER.btcinr": "55E5FEB8-FEA5-416B-88FA-40211541DECA",
"TICKER.ltcjpy": "5AD8E40F-6DF3-489F-9CF1-AF28426A50CF",
"DEPTH.btccad": "5B234CC3-A7C1-47CE-854F-27AEE4CDBDA5",
"TICKER.btcnzd": "5DDD27CA-2466-4D1A-8961-615DEDB68BF1",
"DEPTH.btcgbp": "60C3AF1B-5D40-4D0E-B9FC-CCAB433D2E9C",
"DEPTH.btcnok": "66DA7FB4-6B0C-4A10-9CB7-E2944E046EB5",
"DEPTH.btcthb": "67879668-532F-41F9-8EB0-55E7593A5AB8",
"TICKER.btcsek": "6CAF1244-655B-460F-BEAF-5C56D1F4BEA7",
"TICKER.btcnok": "7532E866-3A03-4514-A4B1-6F86E3A8DC11",
"TICKER.btcgbp": "7B842B7D-D1F9-46FA-A49C-C12F1AD5A533",
"TRADE.LAG": "85174711-BE64-4DE1-B783-0628995D7914",
"DEPTH.btcsek": "8F1FEFAA-7C55-4420-ADA0-4DE15C1C38F3",
"DEPTH.btcdkk": "9219ABB0-B50C-4007-B4D2-51D1711AB19C",
"DEPTH.btcjpy": "94483E07-D797-4DD4-BC72-DC98F1FD39E3",
"TICKER.nmcusd": "9AAEFD15-D101-49F3-A2FD-6B63B85B6BED",
"TICKER.ltcaud": "A046600A-A06C-4EBF-9FFB-BDC8157227E8",
"TICKER.btcjpy": "A39AE532-6A3C-4835-AF8C-DDA54CB4874E",
"DEPTH.btcczk": "A7A970CF-4F6C-4D85-A74E-AC0979049B87",
"TICKER.ltcdkk": "B10A706E-E8C7-4EA8-9148-669F86930B36",
"TICKER.btcpln": "B4A02CB3-2E2D-4A88-AEEA-3C66CB604D01",
"TEST": "BAD99F24-FA8B-4938-BFDF-0C1831FC6665",
"TICKER.btcrub": "BD04F720-3C70-4DCE-AE71-2422AB862C65",
"TICKER.nmcgbp": "BF5126BA-5187-456F-8AE6-963678D0607F",
"TICKER.btckrw": "BF85048D-4DB9-4DBE-9CA3-5B83A1A4186E",
"TICKER.btccny": "C251EC35-56F9-40AB-A4F6-13325C349DE4",
"DEPTH.btcnzd": "CEDF8730-BCE6-4278-B6FE-9BEE42930E95",
"TICKER.btchkd": "D3AE78DD-01DD-4074-88A7-B8AA03CD28DD",
"TICKER.btcthb": "D58E3B69-9560-4B9E-8C58-B5C0F3FDA5E1",
"TICKER.btcusd": "D5F06780-30A8-4A48-A2F8-7ED181B4A13F",
"DEPTH.btcrub": "D6412CA0-B686-464C-891A-D1BA3943F3C6",
"TICKER.nmceur": "D8512D04-F262-4A14-82F2-8E5C96C15E68",
"TRADE.btc": "DBF1DEE9-4F2E-4A08-8CB7-748919A71B21",
"TICKER.nmccad": "DC28033E-7506-484C-905D-1C811A613323",
"DEPTH.btcpln": "E4FF055A-F8BF-407E-AF76-676CAD319A21",
"TICKER.btcdkk": "E5CE0604-574A-4059-9493-80AF46C776B3",
"TICKER.btcaud": "EB6AAA11-99D0-4F64-9E8C-1140872A423D"
}