Mongoose: Exclude objects from results based on another model

Let's say I have a User and Group model, groups have users, like

var GroupSchema = mongoose.Schema({
    name: String,
    users: [{ type: mongoose.Schema.ObjectId, ref: 'User' }]
});

How would I query to get all the Users but exclude the ones that are on Group.users, I'm already doing this by querying first Group then manually filtering against all users

        var groupP = Group.findById(group_id).populate('users');
        var userP = User.find();

        Promise.props({
            group: groupPromise.exec(),
            users: usersPromise.exec()
        })
        .then(function (result) {
        //this gives the expected result but I'm looking for a more straight forward mongoose only solution if possible
          var users = differenceWith(result.users, result.group.users, (a, b) => { return a._id.toString() == b._id.toString()});
        })

via Diego Romero