Mongoose >= 7.0.0 inserts default values after querying with a single field projection and $elemMatch

[guy-evdev]

Prerequisites

• I have written a descriptive issue title

Mongoose version

7.8.0

Node.js version

20.16.0

MongoDB version

5.0.28

MongoDB driver version

5.9.2

Operating system

None

Operating system version (i.e. 20.04, 11.3, 10)

No response

Issue

Before mongoose 7, If we made a query with an $elemMatch projection (on a single subdocument array - no other fields are mentioned in the projection), the document will return with the requested subdocument array item only (as requested) and without any other document fields.

Example Schema:

const MySchema = new Schema({
    details: [{
        name: String,
        id: String,
        age: Number
    }],
    addresses: [{
        streetName: String,
        streetNumber: String
    }],
    permissions: {
        add: {type: Boolean, default: false},
        other: {type: Number, default: 3}
    }
});

Example:

const doc = await Model.findById(some-id, 'details').select({details: {$elemMatch: {id: some-id}}}).exec();

The returned doc will look as follows:

{
    details: [{
        name: 'Gary',
        age: 23,
        id: '12345'
    }]
}

If I try to call doc.permissions it will be undefined

After v7, when using an $elemMatch projection it seems as if the document returns with all the default fields so that when eventually we call a save() action on the document, it saves the document and overrides any field that has a default value

Example:

const doc = await Model.findById(some-id, 'details').select({details: {$elemMatch: {id: some-id}}}).exec();

The returned doc will look as follows:

{
    details: [{
        name: 'Gary',
        age: 23,
        id: '12345'
    }],
    addresses: [],
    permissions: {add: false, other: 3}
}

The addresses subDocument array returns with the default schema values even though it may have multiple items in the array. The same goes for the permissions object which may have an "add" value of true.

If I modify the doc (the specific subdocument array item requested in the projection) and then call save(), the document saves all fields with their default values. It overwrites the current values with the defaults event though the fields were not requested in the projection.

Is this the proper behavior? is there any way to overcome this issue and not get the default values if they are not requested in the projection?

Thanks

[vkarpov15]

I'm unable to repro, the following script shows that Mongoose isn't saving defaults:

const mongoose = require('mongoose');
mongoose.set('debug', true);
console.log(mongoose.version);

const { Schema } = mongoose;

const MySchema = new Schema({
    details: [{
        name: String,
        id: String,
        age: Number
    }],
    addresses: [{
        streetName: String,
        streetNumber: String
    }],
    permissions: {
        add: {type: Boolean, default: false},
        other: {type: Number, default: 3}
    }
});

run().catch(err => console.log(err));

async function run() {
  await mongoose.connect('mongodb://127.0.0.1:27017/mongoose_test');
  const Test = mongoose.model('Test', MySchema);

  await Test.deleteMany({});
  await Test.collection.insertOne({
    details: [{
        name: 'Gary',
        age: 23,
        id: '12345'
    }]
  });

  const doc = await Test.findOne({}, 'details').select({details: {$elemMatch: {id: '12345'}}}).exec();
  console.log(doc);

  await doc.save();
}

The output is as follows.

$ node gh-14893.js 
7.8.0
Mongoose: tests.deleteMany({}, {})
Mongoose: tests.insertOne({ details: [ { name: 'Gary', age: 23, id: '12345' } ] })
Mongoose: tests.findOne({}, { projection: { details: { '$elemMatch': { id: '12345' } } }})
{
  _id: new ObjectId("66e99d4e0470bda98008e099"),
  details: [
    {
      _id: new ObjectId("66e99d4e0470bda98008e09b"),
      name: 'Gary',
      age: 23,
      id: '12345'
    }
  ]
}
Mongoose: tests.findOne({ _id: ObjectId("66e99d4e0470bda98008e099") }, {})
^C

Please modify the script in this post to demonstrate the issue you're seeing.

[guy-evdev]

Please modify the script in this post to demonstrate the issue you're seeing.

Thanks for the reply.
The issue can be reproduced in the following manner:

mongoose.model('ReproductionTest', new Schema({
    field1: {type: Boolean, default: false},
    field2: {
        type: [new Schema({
            subField1: {type: Schema.Types.ObjectId},
            subField2: Boolean,
            subField3: {type: Boolean, default: true},
        })],
        default: undefined
    },
    field3: {type: Number, default: 5},
    field4: [new Schema({
        name: String,
        amount: Number,
        amounts: [{
            amount: Number
        }],
        price: {type: Number, min: 0},
        toDate: Date,
        stockItems: [{
            stockId: Schema.Types.ObjectId,
            itemsPerTicket: {type: Number, default: 1, min: 1}
        }],
    })]
}));

async function run() {
    const Test = mongoose.model('ReproductionTest');
    const id1 = new mongoose.Types.ObjectId();
    const id2 = new mongoose.Types.ObjectId();
    await Test.deleteMany({});
    await Test.insertMany([
        {
            field1: true,
            field2: [
                {
                    subField1: id1,
                    subField2: false,
                    subField3: false
                },
                {
                    subField1: id2,
                    subField2: true
                }
            ],
            field3: 1,
            field4: [
                {
                    name: 'Some name',
                    amount: 100,
                    price: 50,
                }
            ]
        }
    ]);
    let fields;
    const fullDocPreSave = await Test.findOne({}).lean();
    console.log('fullDocPreSave', fullDocPreSave);
    const doc = await Test.findById(fullDocPreSave._id, 'field2 ' + fields)
                          .select({field2: {$elemMatch: {subField1: id1}}})
                          .exec();
    doc.field2.forEach(subDoc => {
        subDoc.subField2 = true;
    });
    await doc.save();
    const fullDocPostSave = await Test.findOne({}).lean();
    console.log('fullDocPostSave', fullDocPostSave);

    await Test.deleteMany({});
}

run();

Please note the following:

In the projection above 'field2 ' + fields the projection is 'field2 undefined'
In mongoose 6, this prevents the query from returning with the default values. When the projection is the 'field2' field only the issue reproduces.
In mongoose 7, the issue reproduces in both projection scenarios.

The outcome in mongoose7

fullDocPreSave {
  _id: new ObjectId("66ea80361da41bb4308e0319"),
  field1: true,
  field2: [
    {
      subField1: new ObjectId("66ea80361da41bb4308e0316"),
      subField2: false,
      subField3: false,
      _id: new ObjectId("66ea80361da41bb4308e031a")
    },
    {
      subField1: new ObjectId("66ea80361da41bb4308e0317"),
      subField2: true,
      subField3: true,
      _id: new ObjectId("66ea80361da41bb4308e031b")
    }
  ],
  field3: 1,
  field4: [
    {
      name: 'Some name',
      amount: 100,
      price: 50,
      _id: new ObjectId("66ea80361da41bb4308e031c"),
      amounts: [],
      stockItems: []
    }
  ],
  __v: 0
}
fullDocPostSave {
  _id: new ObjectId("66ea80361da41bb4308e0319"),
  field1: false,
  field2: [
    {
      subField1: new ObjectId("66ea80361da41bb4308e0316"),
      subField2: true,
      subField3: false,
      _id: new ObjectId("66ea80361da41bb4308e031a")
    },
    {
      subField1: new ObjectId("66ea80361da41bb4308e0317"),
      subField2: true,
      subField3: true,
      _id: new ObjectId("66ea80361da41bb4308e031b")
    }
  ],
  field3: 5,
  field4: [],
  __v: 0
}

This, to me, seems like an ongoing bug since I believe in any scenario the query shouldn't result with the default values when using select and $elemMatch and in any way shouldn't overwrite the doc with default values.

Thanks again

[ilanco]

Hi @vkarpov15 , the details below were obtained by running the above script without a projection.

I've managed to narrow down this behavior to a change in version 6.2.2. Running the above script in version 6.2.1, findOne returns only the requested projection, while on version 6.2.2 it returns the projection and default values for the schema.

6.2.1

Mongoose: tests.findOne({ _id: new ObjectId("66ea9eba6eaca8b4c7a9f7c5") }, { projection: { field2: { '$elemMatch': { subField1: new ObjectId("66ea9eba6eaca8b4c7a9f7c2") } } }})
doc {
  _id: new ObjectId("66ea9eba6eaca8b4c7a9f7c5"),
  field2: [
    {
      subField1: new ObjectId("66ea9eba6eaca8b4c7a9f7c2"),
      subField2: false,
      subField3: false,
      _id: new ObjectId("66ea9eba6eaca8b4c7a9f7c6")
    }
  ]
}

6.2.2

Mongoose: tests.findOne({ _id: new ObjectId("66ea9ebe6e84da91d1db7263") }, { projection: { field2: { '$elemMatch': { subField1: new ObjectId("66ea9ebe6e84da91d1db7260") } } }})
doc {
  field1: false,
  field3: 5,
  _id: new ObjectId("66ea9ebe6e84da91d1db7263"),
  field2: [
    {
      subField1: new ObjectId("66ea9ebe6e84da91d1db7260"),
      subField2: false,
      subField3: false,
      _id: new ObjectId("66ea9ebe6e84da91d1db7264")
    }
  ],
  field4: []
}

[ilanco]

When providing a projection of 'field2 ' + fields which expands fields to undefined the behavior changes in version 7.6.7.

Output using Mongoose 7.6.6

$ node 01377321.js
7.6.6
(node:10874) [DEP0040] DeprecationWarning: The `punycode` module is deprecated. Please use a userland alternative instead.
(Use `node --trace-deprecation ...` to show where the warning was created)
Mongoose: tests.deleteMany({}, {})
Mongoose: tests.insertMany([ { field1: true, field2: [ { subField1: new ObjectId("66eac042ea4c2dd344a670ef"), subField2: false, subField3: false, _id: new ObjectId("66eac042ea4c2dd344a670f3") }, { subField1: new ObjectId("66eac042ea4c2dd344a670f0"), subField2: true, subField3: true, _id: new ObjectId("66eac042ea4c2dd344a670f4") } ], field3: 1, field4: [ { name: 'Some name', amount: 100, price: 50, _id: new ObjectId("66eac042ea4c2dd344a670f5"), amounts: [], stockItems: [] } ], _id: new ObjectId("66eac042ea4c2dd344a670f2"), __v: 0 }], {})
Mongoose: tests.findOne({}, {})
fullDocPreSave {
  _id: new ObjectId("66eac042ea4c2dd344a670f2"),
  field1: true,
  field2: [
    {
      subField1: new ObjectId("66eac042ea4c2dd344a670ef"),
      subField2: false,
      subField3: false,
      _id: new ObjectId("66eac042ea4c2dd344a670f3")
    },
    {
      subField1: new ObjectId("66eac042ea4c2dd344a670f0"),
      subField2: true,
      subField3: true,
      _id: new ObjectId("66eac042ea4c2dd344a670f4")
    }
  ],
  field3: 1,
  field4: [
    {
      name: 'Some name',
      amount: 100,
      price: 50,
      _id: new ObjectId("66eac042ea4c2dd344a670f5"),
      amounts: [],
      stockItems: []
    }
  ],
  __v: 0
}
Mongoose: tests.findOne({ _id: ObjectId("66eac042ea4c2dd344a670f2") }, { projection: { field2: { '$elemMatch': { subField1: ObjectId("66eac042ea4c2dd344a670ef") } }, undefined: 1 }})
doc {
  _id: new ObjectId("66eac042ea4c2dd344a670f2"),
  field2: [
    {
      subField1: new ObjectId("66eac042ea4c2dd344a670ef"),
      subField2: false,
      subField3: false,
      _id: new ObjectId("66eac042ea4c2dd344a670f3")
    }
  ]
}
Mongoose: tests.updateOne({ _id: ObjectId("66eac042ea4c2dd344a670f2"), field2: { '$elemMatch': { subField1: ObjectId("66eac042ea4c2dd344a670ef") } }}, { '$set': { 'field2.$.subField2': true } }, {})
Mongoose: tests.findOne({}, {})
fullDocPostSave {
  _id: new ObjectId("66eac042ea4c2dd344a670f2"),
  field1: true,
  field2: [
    {
      subField1: new ObjectId("66eac042ea4c2dd344a670ef"),
      subField2: true,
      subField3: false,
      _id: new ObjectId("66eac042ea4c2dd344a670f3")
    },
    {
      subField1: new ObjectId("66eac042ea4c2dd344a670f0"),
      subField2: true,
      subField3: true,
      _id: new ObjectId("66eac042ea4c2dd344a670f4")
    }
  ],
  field3: 1,
  field4: [
    {
      name: 'Some name',
      amount: 100,
      price: 50,
      _id: new ObjectId("66eac042ea4c2dd344a670f5"),
      amounts: [],
      stockItems: []
    }
  ],
  __v: 0
}

Output using Mongoose 7.6.7

$ node 01377321.js
7.6.7
(node:10903) [DEP0040] DeprecationWarning: The `punycode` module is deprecated. Please use a userland alternative instead.
(Use `node --trace-deprecation ...` to show where the warning was created)
Mongoose: tests.deleteMany({}, {})
Mongoose: tests.insertMany([ { field1: true, field2: [ { subField1: new ObjectId("66eac048f48feac1f08838f6"), subField2: false, subField3: false, _id: new ObjectId("66eac048f48feac1f08838fa") }, { subField1: new ObjectId("66eac048f48feac1f08838f7"), subField2: true, subField3: true, _id: new ObjectId("66eac048f48feac1f08838fb") } ], field3: 1, field4: [ { name: 'Some name', amount: 100, price: 50, _id: new ObjectId("66eac048f48feac1f08838fc"), amounts: [], stockItems: [] } ], _id: new ObjectId("66eac048f48feac1f08838f9"), __v: 0 }], {})
Mongoose: tests.findOne({}, {})
fullDocPreSave {
  _id: new ObjectId("66eac048f48feac1f08838f9"),
  field1: true,
  field2: [
    {
      subField1: new ObjectId("66eac048f48feac1f08838f6"),
      subField2: false,
      subField3: false,
      _id: new ObjectId("66eac048f48feac1f08838fa")
    },
    {
      subField1: new ObjectId("66eac048f48feac1f08838f7"),
      subField2: true,
      subField3: true,
      _id: new ObjectId("66eac048f48feac1f08838fb")
    }
  ],
  field3: 1,
  field4: [
    {
      name: 'Some name',
      amount: 100,
      price: 50,
      _id: new ObjectId("66eac048f48feac1f08838fc"),
      amounts: [],
      stockItems: []
    }
  ],
  __v: 0
}
Mongoose: tests.findOne({ _id: ObjectId("66eac048f48feac1f08838f9") }, { projection: { field2: { '$elemMatch': { subField1: ObjectId("66eac048f48feac1f08838f6") } }, undefined: 1 }})
doc {
  field1: false,
  field3: 5,
  _id: new ObjectId("66eac048f48feac1f08838f9"),
  field2: [
    {
      subField1: new ObjectId("66eac048f48feac1f08838f6"),
      subField2: false,
      subField3: false,
      _id: new ObjectId("66eac048f48feac1f08838fa")
    }
  ],
  field4: []
}
Mongoose: tests.updateOne({ _id: ObjectId("66eac048f48feac1f08838f9"), field2: { '$elemMatch': { subField1: ObjectId("66eac048f48feac1f08838f6") } }}, { '$set': { 'field2.$.subField2': true, field1: false, field3: 5, field4: [] }}, {})
Mongoose: tests.findOne({}, {})
fullDocPostSave {
  _id: new ObjectId("66eac048f48feac1f08838f9"),
  field1: false,
  field2: [
    {
      subField1: new ObjectId("66eac048f48feac1f08838f6"),
      subField2: true,
      subField3: false,
      _id: new ObjectId("66eac048f48feac1f08838fa")
    },
    {
      subField1: new ObjectId("66eac048f48feac1f08838f7"),
      subField2: true,
      subField3: true,
      _id: new ObjectId("66eac048f48feac1f08838fb")
    }
  ],
  field3: 5,
  field4: [],
  __v: 0
}