kaj*_*jan 2 mongoose mongodb mongoose-populate
我有一个包含动态引用数组的模型。
var postSchema = new Schema({
name: String,
targets: [{
kind: String,
item: { type: ObjectId, refPath: 'targets.kind' }
}]
});
Run Code Online (Sandbox Code Playgroud)
我正在使用目标属性来存储对多个不同模型、用户、线程、附件等的引用。
是否可以只填充我想要的引用?
Post.find({}).populate({
// Does not work
// match: { 'targets.kind': 'Thread' }, // I want to populate only the references that match. ex: Thread, User, Attachment
path: 'targets.item',
model: 'targets.kind',
select: '_id title',
});
Run Code Online (Sandbox Code Playgroud)
谢谢
Nei*_*unn 11
这里的一个重要教训应该mongoose.set('debug', true)是你的新“最好的朋友”。这将显示您正在编写的代码向 MongoDB 发出的实际查询,这非常重要,因为当您真正“看到它”时,它会清除您可能存在的任何误解。
让我们来演示一下为什么您尝试的操作失败了:
const { Schema } = mongoose = require('mongoose');
const uri = 'mongodb://localhost:27017/polypop';
mongoose.set('debug', true);
mongoose.Promise = global.Promise;
const postSchema = new Schema({
name: String,
targets: [{
kind: String,
item: { type: Schema.Types.ObjectId, refPath: 'targets.kind' }
}]
});
const fooSchema = new Schema({
name: String
})
const barSchema = new Schema({
number: Number
});
const Post = mongoose.model('Post', postSchema);
const Foo = mongoose.model('Foo', fooSchema);
const Bar = mongoose.model('Bar', barSchema);
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri, { useNewUrlParser: true });
// Clean all data
await Promise.all(
Object.entries(conn.models).map(([k,m]) => m.deleteMany())
);
// Create some things
let [foo, bar] = await Promise.all(
[{ _t: 'Foo', name: 'Bill' }, { _t: 'Bar', number: 1 }]
.map(({ _t, ...d }) => mongoose.model(_t).create(d))
);
log([foo, bar]);
// Add a Post
let post = await Post.create({
name: 'My Post',
targets: [{ kind: 'Foo', item: foo }, { kind: 'Bar', item: bar }]
});
log(post);
let found = await Post.findOne();
log(found);
let result = await Post.findOne()
.populate({
match: { 'targets.kind': 'Foo' }, // here is the problem!
path: 'targets.item',
});
log(result);
} catch(e) {
console.error(e);
} finally {
mongoose.disconnect();
}
})()
Run Code Online (Sandbox Code Playgroud)
所以那里的评论表明match逻辑有问题,所以让我们看看调试输出,看看为什么:
Mongoose: posts.deleteMany({}, {})
Mongoose: foos.deleteMany({}, {})
Mongoose: bars.deleteMany({}, {})
Mongoose: foos.insertOne({ _id: ObjectId("5bdbc70996ed8e3295b384a0"), name: 'Bill', __v: 0 })
Mongoose: bars.insertOne({ _id: ObjectId("5bdbc70996ed8e3295b384a1"), number: 1, __v: 0 })
[
{
"_id": "5bdbc70996ed8e3295b384a0",
"name": "Bill",
"__v": 0
},
{
"_id": "5bdbc70996ed8e3295b384a1",
"number": 1,
"__v": 0
}
]
Mongoose: posts.insertOne({ _id: ObjectId("5bdbc70996ed8e3295b384a2"), name: 'My Post', targets: [ { _id: ObjectId("5bdbc70996ed8e3295b384a4"), kind: 'Foo', item: ObjectId("5bdbc70996ed8e3295b384a0") }, { _id: ObjectId("5bdbc70996ed8e3295b384a3"), kind: 'Bar', item: ObjectId("5bdbc70996ed8e3295b384a1") } ], __v: 0 })
{
"_id": "5bdbc70996ed8e3295b384a2",
"name": "My Post",
"targets": [
{
"_id": "5bdbc70996ed8e3295b384a4",
"kind": "Foo",
"item": {
"_id": "5bdbc70996ed8e3295b384a0",
"name": "Bill",
"__v": 0
}
},
{
"_id": "5bdbc70996ed8e3295b384a3",
"kind": "Bar",
"item": {
"_id": "5bdbc70996ed8e3295b384a1",
"number": 1,
"__v": 0
}
}
],
"__v": 0
}
Mongoose: posts.findOne({}, { projection: {} })
{
"_id": "5bdbc70996ed8e3295b384a2",
"name": "My Post",
"targets": [
{
"_id": "5bdbc70996ed8e3295b384a4",
"kind": "Foo",
"item": "5bdbc70996ed8e3295b384a0"
},
{
"_id": "5bdbc70996ed8e3295b384a3",
"kind": "Bar",
"item": "5bdbc70996ed8e3295b384a1"
}
],
"__v": 0
}
Mongoose: posts.findOne({}, { projection: {} })
Mongoose: bars.find({ 'targets.kind': 'Foo', _id: { '$in': [ ObjectId("5bdbc70996ed8e3295b384a1") ] } }, { projection: {} })
Mongoose: foos.find({ 'targets.kind': 'Foo', _id: { '$in': [ ObjectId("5bdbc70996ed8e3295b384a0") ] } }, { projection: {} })
{
"_id": "5bdbc70996ed8e3295b384a2",
"name": "My Post",
"targets": [
{
"_id": "5bdbc70996ed8e3295b384a4",
"kind": "Foo",
"item": null
},
{
"_id": "5bdbc70996ed8e3295b384a3",
"kind": "Bar",
"item": null
}
],
"__v": 0
}
Run Code Online (Sandbox Code Playgroud)
这是显示其他所有内容实际上都在工作的完整输出,事实上,如果没有,match您将获得这些项目的填充数据。但是仔细看看向foo和bar集合发出的两个查询:
Mongoose: bars.find({ 'targets.kind': 'Foo', _id: { '$in': [ ObjectId("5bdbc70996ed8e3295b384a1") ] } }, { projection: {} })
Mongoose: foos.find({ 'targets.kind': 'Foo', _id: { '$in': [ ObjectId("5bdbc70996ed8e3295b384a0") ] } }, { projection: {} })
Run Code Online (Sandbox Code Playgroud)
因此'targets.kind',match实际上是在foo和bar集合上搜索您包含在下面的,而不是posts像您所期望的那样在集合中搜索。连同输出的其余部分,这应该让您了解populate()实际工作方式,因为没有任何内容专门返回kind: 'Foo'示例中的“数组条目” 。
这种“过滤数组”的过程实际上甚至不是“真正”的自然 MongoDB 查询,除了“第一次和单数匹配”之外,您实际上通常会使用.aggregate()和$filter运算符。您可以通过位置$运算符获得“单数”,但如果您想要“所有 foos”,其中有多个,那么它需要$filter取而代之。
所以这里真正的核心问题populate()实际上是“过滤数组”的错误位置和错误操作。相反,您真的想“聪明地”只返回您想要的数组条目,然后再执行任何其他操作来“填充”项目。
从上面的清单中注意到问题中所暗示的寓意,为了“加入”并获得整体结果,引用了“多个模型”。虽然这在“RDBMS 领域”中似乎是合乎逻辑的,但对于 MongoDB 和“文档数据库”的一般“同类”来说,这样做肯定不是这种情况,也不实际或有效。
这里要记住的关键是“集合”中的“文档”不必都具有与 RDBMS 相同的“表结构”。结构可以变化,虽然最好不要“变化很大”,但将“多态对象”存储在单个集合中当然是非常有效的。毕竟,您实际上想要将所有这些东西引用回同一个父级,那么为什么它们需要在不同的集合中呢?简而言之,它们根本不需要:
const { Schema } = mongoose = require('mongoose');
const uri = 'mongodb://localhost:27017/polypop';
mongoose.set('debug', true);
mongoose.Promise = global.Promise;
const postSchema = new Schema({
name: String,
targets: [{
kind: String,
item: { type: Schema.Types.ObjectId, ref: 'Target' }
}]
});
const targetSchema = new Schema({});
const fooSchema = new Schema({
name: String
});
const barSchema = new Schema({
number: Number
});
const bazSchema = new Schema({
title: String
});
const log = data => console.log(JSON.stringify(data, undefined, 2));
const Post = mongoose.model('Post', postSchema);
const Target = mongoose.model('Target', targetSchema);
const Foo = Target.discriminator('Foo', fooSchema);
const Bar = Target.discriminator('Bar', barSchema);
const Baz = Target.discriminator('Baz', bazSchema);
(async function() {
try {
const conn = await mongoose.connect(uri,{ useNewUrlParser: true });
// Clean data - bit hacky but just a demo
await Promise.all(
Object.entries(conn.models).map(([k, m]) => m.deleteMany() )
);
// Insert some things
let [foo1, bar, baz, foo2] = await Promise.all(
[
{ _t: 'Foo', name: 'Bill' },
{ _t: 'Bar', number: 1 },
{ _t: 'Baz', title: 'Title' },
{ _t: 'Foo', name: 'Ted' }
].map(({ _t, ...d }) => mongoose.model(_t).create(d))
);
log([foo1, bar, baz, foo2]);
// Add a Post
let post = await Post.create({
name: 'My Post',
targets: [
{ kind: 'Foo', item: foo1 },
{ kind: 'Bar', item: bar },
{ kind: 'Baz', item: baz },
{ kind: 'Foo', item: foo2 }
]
});
log(post);
let found = await Post.findOne();
log(found);
let result1 = await Post.findOne()
.populate({
path: 'targets.item',
match: { __t: 'Foo' }
});
log(result1);
let result2 = await Post.aggregate([
// Only get documents with a matching entry
{ "$match": {
"targets.kind": "Foo"
}},
// Optionally filter the array
{ "$addFields": {
"targets": {
"$filter": {
"input": "$targets",
"cond": {
"$eq": [ "$$this.kind", "Foo" ]
}
}
}
}},
// Lookup from single source
{ "$lookup": {
"from": Target.collection.name,
"localField": "targets.item",
"foreignField": "_id",
"as": "matches"
}},
// Marry up arrays
{ "$project": {
"name": 1,
"targets": {
"$map": {
"input": "$targets",
"in": {
"kind": "$$this.kind",
"item": {
"$arrayElemAt": [
"$matches",
{ "$indexOfArray": [ "$matches._id", "$$this.item" ] }
]
}
}
}
}
}}
]);
log(result2);
let result3 = await Post.aggregate([
// Only get documents with a matching entry
{ "$match": {
"targets.kind": "Foo"
}},
// Optionally filter the array
{ "$addFields": {
"targets": {
"$filter": {
"input": "$targets",
"cond": {
"$eq": [ "$$this.kind", "Foo" ]
}
}
}
}},
// Lookup from single source with overkill of type check
{ "$lookup": {
"from": Target.collection.name,
"let": { "targets": "$targets" },
"pipeline": [
{ "$match": {
"$expr": {
"$in": [ "$_id", "$$targets.item" ]
},
"__t": "Foo"
}}
],
"as": "matches"
}},
// Marry up arrays
{ "$project": {
"name": 1,
"targets": {
"$map": {
"input": "$targets",
"in": {
"kind": "$$this.kind",
"item": {
"$arrayElemAt": [
"$matches",
{ "$indexOfArray": [ "$matches._id", "$$this.item" ] }
]
}
}
}
}
}}
]);
console.log(result3);
} catch(e) {
console.error(e);
} finally {
mongoose.disconnect();
}
})()
Run Code Online (Sandbox Code Playgroud)
这有点长,还有更多的概念需要解决,但基本原则是,我们只使用一个,而不是对不同类型使用“多个集合” 。用于此的“猫鼬”方法在模型设置中使用“鉴别器”,这与这部分代码相关:
const Post = mongoose.model('Post', postSchema);
const Target = mongoose.model('Target', targetSchema);
const Foo = Target.discriminator('Foo', fooSchema);
const Bar = Target.discriminator('Bar', barSchema);
const Baz = Target.discriminator('Baz', bazSchema);
Run Code Online (Sandbox Code Playgroud)
这实际上只是.discriminator()从“单一”集合的“基本模型”调用而不是调用mongoose.model(). 关于这个真正的好东西是只要你的代码的其余部分而言,Baz和Bar等等都只是当作一个“样板”透明的,但它们实际上是做一些真正冷静下。
所以所有这些“相关的东西”(它们确实是,即使你还不这么认为)实际上都保存在同一个集合中,但是使用各个模型的操作会考虑一个“自动”kind键。这是__t默认设置,但您实际上可以在选项中指定您想要的任何内容。
这些实际上都在同一个集合中这一事实非常重要,因为您基本上可以很容易地为不同类型的数据查询相同的集合。简单地说:
Foo.find({})
Run Code Online (Sandbox Code Playgroud)
实际上会打电话
targets.find({ __t: 'Foo' })
Run Code Online (Sandbox Code Playgroud)
并自动执行此操作。但更重要的是
Target.find({ __t: { "$in": [ 'Foo', 'Baz' ] } })
Run Code Online (Sandbox Code Playgroud)
将从具有“单一请求”的“单一集合”返回所有预期结果。
所以看看populate()这个结构下的修改:
let result1 = await Post.findOne()
.populate({
path: 'targets.item',
match: { __t: 'Foo' }
});
log(result1);
Run Code Online (Sandbox Code Playgroud)
这显示在日志中:
Mongoose: posts.findOne({}, { projection: {} })
Mongoose: targets.find({ __t: 'Foo', _id: { '$in': [ ObjectId("5bdbe2895b1b843fba050569"), ObjectId("5bdbe2895b1b843fba05056a"), ObjectId("5bdbe2895b1b843fba05056b"), ObjectId("5bdbe2895b1b843fba05056c") ] } }, { projection: {} })
Run Code Online (Sandbox Code Playgroud)
请注意,即使所有“四个”相关ObjectId值都随请求一起发送,附加约束__t: 'Foo'也绑定了实际返回和合并的文档。结果然后变得不言自明,因为只有'Foo'条目被填充。但还要注意“捕获”:
{
"_id": "5bdbe2895b1b843fba05056d",
"name": "My Post",
"targets": [
{
"_id": "5bdbe2895b1b843fba050571",
"kind": "Foo",
"item": {
"__t": "Foo",
"_id": "5bdbe2895b1b843fba050569",
"name": "Bill",
"__v": 0
}
},
{
"_id": "5bdbe2895b1b843fba050570",
"kind": "Bar",
"item": null
},
{
"_id": "5bdbe2895b1b843fba05056f",
"kind": "Baz",
"item": null
},
{
"_id": "5bdbe2895b1b843fba05056e",
"kind": "Foo",
"item": {
"__t": "Foo",
"_id": "5bdbe2895b1b843fba05056c",
"name": "Ted",
"__v": 0
}
}
],
"__v": 0
}
Run Code Online (Sandbox Code Playgroud)
这实际上是一个更长的主题,并且在其他地方得到了更完整的回答,但如上面的输出所示,这里的基础知识populate()实际上仍然完全没有将数组中的结果“过滤”为仅所需的匹配项。
另一件事是,populate()从“性能”的角度来看,这真的不是一个好主意,因为真正发生的是“另一个查询”(在我们的第二种形式中,我们优化为仅一个)或“许多查询”,具体取决于您的结构实际上正在发送到数据库,并且正在客户端上一起重建结果。
总体而言,您最终会返回比实际需要多得多的数据,并且充其量只能依靠手动客户端过滤来丢弃那些不需要的结果。所以“理想”的位置是让“服务器”做那种事情,并且只返回你真正需要的数据。
populate()很久以前,该方法是作为“方便”添加到 mongoose API 中的。从那以后,MongoDB 不断发展,现在$lookup作为一种“本机”方式在服务器上通过单个请求执行“加入”。
有不同的方法可以做到这一点,但只是触及与现有populate()功能密切相关的“两个”,但有改进:
let result2 = await Post.aggregate([
// Only get documents with a matching entry
{ "$match": {
"targets.kind": "Foo"
}},
// Optionally filter the array
{ "$addFields": {
"targets": {
"$filter": {
"input": "$targets",
"cond": {
"$eq": [ "$$this.kind", "Foo" ]
}
}
}
}},
// Lookup from single source
{ "$lookup": {
"from": Target.collection.name,
"localField": "targets.item",
"foreignField": "_id",
"as": "matches"
}},
// Marry up arrays
{ "$project": {
"name": 1,
"targets": {
"$map": {
"input": "$targets",
"in": {
"kind": "$$this.kind",
"item": {
"$arrayElemAt": [
"$matches",
{ "$indexOfArray": [ "$matches._id", "$$this.item" ] }
]
}
}
}
}
}}
]);
log(result2);
Run Code Online (Sandbox Code Playgroud)
$filter为了从数组中“预先丢弃”实际上与我们想要的类型不匹配的项目,使用了两个基本的“优化” 。这可以是完全可选的,稍后会详细介绍,但在可能的情况下,这可能是一件好事,因为我们甚至不会_id在外部集合中寻找匹配的值,而不是'Foo'事物。
另一个当然是它$lookup本身,这意味着我们实际上只是进行一次往返服务器而不是单独的往返,并且在返回任何响应之前完成“加入”。在这里,我们只是_id在外部集合中查找与target.items数组条目值匹配的值。我们已经为 过滤了那些'Foo',所以这就是返回的全部内容:
{
"_id": "5bdbe6aa2c4a2240c16802e2",
"name": "My Post",
"targets": [
{
"kind": "Foo",
"item": {
"_id": "5bdbe6aa2c4a2240c16802de",
"__t": "Foo",
"name": "Bill",
"__v": 0
}
},
{
"kind": "Foo",
"item": {
"_id": "5bdbe6aa2c4a2240c16802e1",
"__t": "Foo",
"name": "Ted",
"__v": 0
}
}
]
}
Run Code Online (Sandbox Code Playgroud)
对于“轻微”变化,我们实际上甚至可以使用 MongoDB 3.6 及更高版本的“子管道”处理来检查表达式中的__t值$lookup。这里的主要用例是,如果您选择kind从父项中Post完全删除 ,并仅依赖存储中使用的鉴别器引用固有的“种类”信息:
let result3 = await Post.aggregate([
// Only get documnents with a matching entry
{ "$match": {
"targets.kind": "Foo"
}},
// Optionally filter the array
{ "$addFields": {
"targets": {
"$filter": {
"input": "$targets",
"cond": {
"$eq": [ "$$this.kind", "Foo" ]
}
}
}
}},
// Lookup from single source with overkill of type check
{ "$lookup": {
"from": Target.collection.name,
"let": { "targets": "$targets" },
"pipeline": [
{ "$match": {
"$expr": {
"$in": [ "$_id", "$$targets.item" ]
},
"__t": "Foo"
}}
],
"as": "matches"
}},
// Marry up arrays
{ "$project": {
"name": 1,
"targets": {
"$map": {
"input": "$targets",
"in": {
"kind": "$$this.kind",
"item": {
"$arrayElemAt": [
"$matches",
{ "$indexOfArray": [ "$matches._id", "$$this.item" ] }
]
}
}
}
}
}}
]);
log(result3);
Run Code Online (Sandbox Code Playgroud)
这具有相同的“过滤”结果,并且类似于“单一请求”和“单一响应”。
整个主题变得更宽泛一些,尽管聚合管道可能看起来比简单的populate()调用要笨拙得多,但编写一个可以从模型中抽象出来并几乎生成所需的大部分数据结构代码的包装器是相当简单的。您可以在“在 Mongoose 中填充后查询”中看到对此的概述,这本质上与您在我们解决“多集合连接”的初始问题以及为什么您真的不这样做后基本上在这里问的问题相同需要他们。
这里需要注意的是,$lookup实际上无法“动态”确定要“加入”哪个集合。您需要像这里所做的那样静态地包含该信息,因此这是实际偏爱“鉴别器”而不是使用多个集合的另一个原因。这不仅是“更好的性能”,而且实际上是性能最高的选项实际上支持您尝试做的事情的唯一方式。
作为参考,第二个列表的“完整”(由于最大帖子长度而被截断)输出将是:
Mongoose: posts.deleteMany({}, {})
Mongoose: targets.deleteMany({}, {})
Mongoose: targets.deleteMany({}, {})
Mongoose: targets.deleteMany({}, {})
Mongoose: targets.deleteMany({}, {})
Mongoose: targets.insertOne({ _id: ObjectId("5bdbe2895b1b843fba050569"), __t: 'Foo', name: 'Bill', __v: 0 })
Mongoose: targets.insertOne({ _id: ObjectId("5bdbe2895b1b843fba05056a"), __t: 'Bar', number: 1, __v: 0 })
Mongoose: targets.insertOne({ _id: ObjectId("5bdbe2895b1b843fba05056b"), __t: 'Baz', title: 'Title', __v: 0 })
Mongoose: targets.insertOne({ _id: ObjectId("5bdbe2895b1b843fba05056c"), __t: 'Foo', name: 'Ted', __v: 0 })
[
{
"_id": "5bdbe2895b1b843fba050569",
"__t": "Foo",
"name": "Bill",
"__v": 0
},
{
"_id": "5bdbe2895b1b843fba05056a",
"__t": "Bar",
"number": 1,
"__v": 0
},
{
"_id": "5bdbe2895b1b843fba05056b",
"__t": "Baz",
"title": "Title",
"__v": 0
},
{
"_id": "5bdbe2895b1b843fba05056c",
"__t": "Foo",
"name": "Ted",
"__v": 0
}
]
Mongoose: posts.insertOne({ _id: ObjectId("5bdbe2895b1b843fba05056d"), name: 'My Post', targets: [ { _id: ObjectId("5bdbe2895b1b843fba050571"), kind: 'Foo', item: ObjectId("5bdbe2895b1b843fba050569") }, { _id: ObjectId("5bdbe2895b1b843fba050570"), kind: 'Bar', item: ObjectId("5bdbe2895b1b843fba05056a") }, { _id: ObjectId("5bdbe2895b1b843fba05056f"), kind: 'Baz', item: ObjectId("5bdbe2895b1b843fba05056b") }, { _id: ObjectId("5bdbe2895b1b843fba05056e"), kind: 'Foo', item: ObjectId("5bdbe2895b1b843fba05056c") } ], __v: 0 })
{
"_id": "5bdbe2895b1b843fba05056d",
"name": "My Post",
"targets": [
{
"_id": "5bdbe2895b1b843fba050571",
"kind": "Foo",
"item": {
"_id": "5bdbe2895b1b843fba050569",
"__t": "Foo",
"name": "Bill",
"__v": 0
}
},
{
"_id": "5bdbe2895b1b843fba050570",
"kind": "Bar",
"item": {
"_id": "5bdbe2895b1b843fba05056a",
"__t": "Bar",
"number": 1,
"__v": 0
}
},
{
"_id": "5bdbe2895b1b843fba05056f",
"kind": "Baz",
"item": {
"_id": "5bdbe2895b1b843fba05056b",
"__t": "Baz",
"title": "Title",
"__v": 0
}
},
{
"_id": "5bdbe2895b1b843fba05056e",
"kind": "Foo",
"item": {
"_id": "5bdbe2895b1b843fba05056c",
"__t": "Foo",
"name": "Ted",
"__v": 0
}
}
],
"__v": 0
}
Mongoose: posts.findOne({}, { projection: {} })
{
"_id": "5bdbe2895b1b843fba05056d",
"name": "My Post",
"targets": [
{
"_id": "5bdbe2895b1b843fba050571",
"kind": "Foo",
"item": "5bdbe2895b1b843fba050569"
},
{
"_id": "5bdbe2895b1b843fba050570",
"kind": "Bar",
"item": "5bdbe2895b1b843fba05056a"
},
{
"_id": "5bdbe2895b1b843fba05056f",
"kind": "Baz",
"item": "5bdbe2895b1b843fba05056b"
},
{
"_id": "5bdbe2895b1b843fba05056e",
"kind": "Foo",
"item": "5bdbe2895b1b843fba05056c"
}