How do I FILTER returned data between two dates from mongodb using an aggregation : match, lookup and project?

Question

I would like to get all Notifications between a start and end date that are related to a Register when I pass in a userId.

Register Schema

const mongoose = require('mongoose');
const Schema = mongoose.Schema;
const RegisterSchema = new Schema({
    userId: {type: Schema.Types.ObjectId, required: true},
    accessToken: {type:String, required: true, default: null},
})
module.exports = Register = mongoose.model( 'register', RegisterSchema)

Here is some register data

[
  {
    "_id": "5eac9e815fc57b07f5d0d29f",
    "userId": "5ea108babb65b800172b11be",
    "accessToken": "111"
  },
  {
    "_id": "5ecaeba3c7b910d3276df839",
    "userId": "5e6c2dddad72870c84f8476b",
    "accessToken": "222"
  }
]

The next document contains data that is related to the Register schema via the accessToken

Notifications

const mongoose = require('mongoose');
const Schema = mongoose.Schema;
const NotificationSchema = new Schema({
    accessToken: {type:String, required: true},
    summaryId: {type:Number, required: true},
    dateCreated: {type: Date, default: Date.now},
})
module.exports = Notification = mongoose.model( 'notification', NotificationSchema)

Here is some notification data

[{
    "_id": "5ebf0390c719e60004f42e74",
    "accessToken": "111",
    "summaryId": 1111,
    "dateCreated": "17 Apr 2020" }, 
  {
    "_id": "6ebf0390c719e60004f42e76",
    "accessToken": "222",
    "summaryId": 2221,
    "dateCreated": "18 Apr 2020" },
  {
    "_id": "6ebf0390c719e60004f42e78",
    "accessToken": "111",
    "summaryId": 1112,
    "dateCreated": "25 May 2020" },
  {
    "_id": "6ebf0390c719e60004f42e80",
    "accessToken": "222",
    "summaryId": 2222,
    "dateCreated": "26 May 2020" }
]

Try 1

        var userId = '5ea108babb65b800172b11be'
        var dateStart = '27 Apr 2020';
        var dateEnd   = '27 May 2020'; 

        var match = {$match: { userId: mongoose.Types.ObjectId(userId) } };

        var lookup ={
            $lookup:
            {
                from: "notifications",
                localField: "accessToken",
                foreignField: "accessToken",
                as: "testingThis"
            }
        };

        project = {
            $project: {
                items: {
                    $filter: {
                    input: "$items",
                    as: "item",
                    cond: { {"dateCreated": {'$gte': dateStart, '$lte': dateEnd }} }
                    }
                }
            }
        };

        var agg = [
            match,
            lookup,
            project
        ];


        Register.aggregate(agg)
        .then( events => {
            if(events){
                return resolve(events);
            }else{
                return reject({success:false});
            }
        })
        .catch(err => {
            console.log('ERROR ' + JSON.stringify(err.message));
            return reject({success:false});
        })  

Try 1 Error

I am expecting to see the notification for 25 May for accessToken of 111, but I am getting an error:

ERROR : {"\"An object representing an expression must have exactly one field: { $gte: new Date(1588017802546), $lte: new Date(1590609802546) }\""}

Try 2

I got rid of the error ... but still getting nothing returned:

        var dateCondition = { $and: [
            { $gte: [ "$$item.dateCreated", dateStart.getTime() ] },
            { $lte: [ "$$item.dateCreated", dateEnd.getTime() ] }
          ] }

          project = {
            $project: {
                items: {
                    $filter: {
                    input: "$items",
                    as: "item",
                    cond: dateCondition
                    }
                }
            }
        };

This is what my project looks like:

{
  "$project": {
    "items": {
      "$filter": {
        "input": "$items",
        "as": "item",
        "cond": {
          "$and": [
            {"$gte": ["$$item.dateCreated",1588019227296] },
            {"$lte": ["$$item.dateCreated",1590611227296] }
          ] } } } }
}

Try 3

using advice from comments... I changed 'items' (from try 2) to 'notifications'

        var dateCondition = { $and: [
            { $gte: [ "$$item.dateCreated", dateStart.getTime() ] },
            { $lte: [ "$$item.dateCreated", dateEnd.getTime() ] }
          ] }

          project = {
            $project: {
                notifications: {
                    $filter: {
                    input: "$notifications",
                    as: "item",
                    cond: dateCondition
                    }
                }
            }
        };

still does not work

so in an attempt to simplify as much as I can to get this to work... I am trying it with summary id

Try 4

          dateCondition = { $and: [
            { $gte: [ "$$item.summaryId", 1 ] },
            { $lte: [ "$$item.summaryId", 555555 ] }
          ] }          

          project = {
            $project: {
                notifications: {
                    $filter: {
                    input: "$notifications",
                    as: "item",
                    cond: dateCondition
                    }
                }
            }
        };

which works... so that leads me to think it is a date problem.

Final Code - works!

        // make sure the input dates are REALLY date objects
        var dateStart = new Date(inputDateStart);
        var dateEnd = new Date(inputDateEnd);     

        var match = {$match: { userId: mongoose.Types.ObjectId(userId) } };

        var lookup ={
            $lookup:
            {
                from: "my_Notifications",
                localField: "accessToken",
                foreignField: "accessToken",
                as: "notifications"
            }
        };

        var dateCondition = { $and: [
            { $gte: [ "$$item.dateCreated", dateStart ] },
            { $lte: [ "$$item.dateCreated", dateEnd ] }
          ]}  

        project = {
            $project: {
                notifications: {
                    $filter: {
                    input: "$notifications",
                    as: "item",
                    cond: dateCondition
                    } } }
        };

        var agg = [
            match,
            lookup,
            project
        ];

        Register.aggregate(agg)
        .then( ..... )

Answer 1

Your solution looks almost right, provided that dateStart and dateStart are actually Date objects and not Strings.

Your Try 2 was incomplete I'm not sure it uses the $lookup from Try 1 or not. If so you have to make sure the output of $lookup is the same as input of $filter. So you should change as in $lookup to match input of $filter

{
  $lookup: {
    from: "notifications",
    localField: "accessToken",
    foreignField: "accessToken",
    as: "items" // here
  }

}

Alternative Solution

I'm not sure what you want as output. If you only need array of notifications without the user object, you can try the following.

[{
  $match: { userId: mongoose.Types.ObjectId(userId) }
}, {
  $lookup: {
    from: "notifications",
    localField: "accessToken", // don't forget to index register.accessToken
    foreignField: "accessToken", // don't forget to index notification.accessToken
    as: "notifications"
  }
}, {
  $unwind: "$notifications"
}, {
  $match: { 
    dateCreated: { $gte: dateStart, $lte: dateEnd } // dateStart, dateEnd should be Date objects
  }
}, { // optional, move notifications to top lvel
  $replaceRoot: { root: '$notifications' }
}]