Chapter 5 Indexes
In this chapter we will cover:
- What indexes are and why you'd want to use them.
- How to choose which fields to index
- How to enforce and evaluate index usage
- Administrative details on creating and removing indexes.
Introduction to Indexes
A query that does not use an index is called a collection scan,which mean that the server has to "look through the whole book" to find query's results.
>
> for (i=0;i<1000000;i++){
... db.users.insertOne(
... {
... "i":i,
... "username":"user"+i,
... "age": Math.floor(Math.random()*120),
... "created": new Date()
... }
... );
... }
{
"acknowledged" : true,
"insertedId" : ObjectId("624d9142e5e7e0a95c1ee18a")
}
>
> db.users.find({"username": "user101"}).explain("executionStats")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "users.users",
"indexFilterSet" : false,
"parsedQuery" : {
"username" : {
"$eq" : "user101"
}
},
"winningPlan" : {
"stage" : "COLLSCAN",
"filter" : {
"username" : {
"$eq" : "user101"
}
},
"direction" : "forward"
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 808,
"totalKeysExamined" : 0,
"totalDocsExamined" : 1000001,
"executionStages" : {
"stage" : "COLLSCAN",
"filter" : {
"username" : {
"$eq" : "user101"
}
},
"nReturned" : 1,
"executionTimeMillisEstimate" : 71,
"works" : 1000003,
"advanced" : 1,
"needTime" : 1000001,
"needYield" : 0,
"saveState" : 1000,
"restoreState" : 1000,
"isEOF" : 1,
"direction" : "forward",
"docsExamined" : 1000001
}
},
"serverInfo" : {
"host" : "DBAMAXWELL",
"port" : 27017,
"version" : "4.4.13",
"gitVersion" : "df25c71b8674a78e17468f48bcda5285decb9246"
},
"ok" : 1
}
>
>
> db.users.createIndex({"username": 1})
{
"createdCollectionAutomatically" : false,
"numIndexesBefore" : 1,
"numIndexesAfter" : 2,
"ok" : 1
}
> db.users.find({"username": "user101"}).explain("executionStats")
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "users.users",
"indexFilterSet" : false,
"parsedQuery" : {
"username" : {
"$eq" : "user101"
}
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"username" : 1
},
"indexName" : "username_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"username" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"username" : [
"[\"user101\", \"user101\"]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 1,
"executionTimeMillis" : 1,
"totalKeysExamined" : 1,
"totalDocsExamined" : 1,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 2,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"docsExamined" : 1,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 1,
"executionTimeMillisEstimate" : 0,
"works" : 2,
"advanced" : 1,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"keyPattern" : {
"username" : 1
},
"indexName" : "username_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"username" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"username" : [
"[\"user101\", \"user101\"]"
]
},
"keysExamined" : 1,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
},
"serverInfo" : {
"host" : "DBAMAXWELL",
"port" : 27017,
"version" : "4.4.13",
"gitVersion" : "df25c71b8674a78e17468f48bcda5285decb9246"
},
"ok" : 1
}
>
Introduction to Compound Indexes
The purpose of an index is to make your queries as efficient as possible.For many query patterns it is necessary to build indexes based on two or more keys.
>
> db.users.find().sort({"age": 1,"username": 1})
Error: error: {
"ok" : 0,
"errmsg" : "Executor error during find command :: caused by :: Sort exceeded memory limit of 104857600 bytes, but did not opt in to external sorting.",
"code" : 292,
"codeName" : "QueryExceededMemoryLimitNoDiskUseAllowed"
}
> db.users.createIndex({"age": 1, "username": 1})
{
"createdCollectionAutomatically" : false,
"numIndexesBefore" : 2,
"numIndexesAfter" : 3,
"ok" : 1
}
This is called a compound index and is useful if your query has multiple sort directions or multiplle keys in the criteria.A compound index is an index on more than one field.
> db.users.find({},{"_id": 0, "i": 0, "created": 0})
{ "name" : "joe", "age" : 30, "sex" : "male", "location" : "Wisconsin" }
{ "username" : "user0", "age" : 93 }
{ "username" : "user1", "age" : 98 }
{ "username" : "user2", "age" : 61 }
{ "username" : "user3", "age" : 58 }
{ "username" : "user4", "age" : 67 }
{ "username" : "user5", "age" : 8 }
{ "username" : "user6", "age" : 93 }
{ "username" : "user7", "age" : 14 }
{ "username" : "user8", "age" : 58 }
{ "username" : "user9", "age" : 77 }
{ "username" : "user10", "age" : 97 }
{ "username" : "user11", "age" : 53 }
{ "username" : "user12", "age" : 57 }
{ "username" : "user13", "age" : 94 }
{ "username" : "user14", "age" : 20 }
{ "username" : "user15", "age" : 87 }
{ "username" : "user16", "age" : 57 }
{ "username" : "user17", "age" : 17 }
{ "username" : "user18", "age" : 50 }
Type "it" for more
>
The way MongoDB uses this index depends on the type of query you're doing.These are the three most common ways:
- This is an equality query, which searches for a single value.
this type of query is very efficient: MongoDB can jump directly to do the correct and doesn't need to sort the results because traversing the index returns the data in the correct order.
> db.users.find({"age": 21}).sort({"username": -1})
{ "_id" : ObjectId("624d9141e5e7e0a95c1ee129"), "i" : 999902, "username" : "user999902", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.954Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1ee091"), "i" : 999750, "username" : "user999750", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.764Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1ee086"), "i" : 999739, "username" : "user999739", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.754Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1ee07b"), "i" : 999728, "username" : "user999728", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.735Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1ee051"), "i" : 999686, "username" : "user999686", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.695Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1ee04a"), "i" : 999679, "username" : "user999679", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.690Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1ee017"), "i" : 999628, "username" : "user999628", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.621Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1edfb3"), "i" : 999528, "username" : "user999528", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.301Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1edf9b"), "i" : 999504, "username" : "user999504", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.274Z") }
{ "_id" : ObjectId("624d9141e5e7e0a95c1edef4"), "i" : 999337, "username" : "user999337", "age" : 21, "created" : ISODate("2022-04-06T13:10:25.038Z") }
{ "_id" : ObjectId("624d9140e5e7e0a95c1edebe"), "i" : 999283, "username" : "user999283", "age" : 21, "created" : ISODate("2022-04-06T13:10:24.974Z") }
{ "_id" : ObjectId("624d9140e5e7e0a95c1ede64"), "i" : 999193, "username" : "user999193", "age" : 21, "created" : ISODate("2022-04-06T13:10:24.874Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112585"), "i" : 99898, "username" : "user99898", "age" : 21, "created" : ISODate("2022-04-06T12:51:59.448Z") }
{ "_id" : ObjectId("624d9140e5e7e0a95c1edcd7"), "i" : 998796, "username" : "user998796", "age" : 21, "created" : ISODate("2022-04-06T13:10:24.397Z") }
{ "_id" : ObjectId("624d9140e5e7e0a95c1edc7b"), "i" : 998704, "username" : "user998704", "age" : 21, "created" : ISODate("2022-04-06T13:10:24.285Z") }
{ "_id" : ObjectId("624d9140e5e7e0a95c1edc1e"), "i" : 998611, "username" : "user998611", "age" : 21, "created" : ISODate("2022-04-06T13:10:24.147Z") }
{ "_id" : ObjectId("624d913fe5e7e0a95c1edba7"), "i" : 998492, "username" : "user998492", "age" : 21, "created" : ISODate("2022-04-06T13:10:23.983Z") }
{ "_id" : ObjectId("624d913fe5e7e0a95c1ed9cd"), "i" : 998018, "username" : "user998018", "age" : 21, "created" : ISODate("2022-04-06T13:10:23.356Z") }
{ "_id" : ObjectId("624d913fe5e7e0a95c1ed9ae"), "i" : 997987, "username" : "user997987", "age" : 21, "created" : ISODate("2022-04-06T13:10:23.327Z") }
{ "_id" : ObjectId("624d913fe5e7e0a95c1ed9ac"), "i" : 997985, "username" : "user997985", "age" : 21, "created" : ISODate("2022-04-06T13:10:23.325Z") }
Type "it" for more
>
- This is range query, which looks for documents matching multiple values(in this case,all ages between 21 and 30).MongoDB will use the first key in the index,"age", to return the matching documents.
>
> db.users.find({"age": {"$gte": 21, "$lte": 30}})
{ "_id" : ObjectId("624d8cefe5e7e0a95c11260a"), "i" : 100031, "username" : "user100031", "age" : 21, "created" : ISODate("2022-04-06T12:51:59.622Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c1126da"), "i" : 100239, "username" : "user100239", "age" : 21, "created" : ISODate("2022-04-06T12:51:59.864Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112720"), "i" : 100309, "username" : "user100309", "age" : 21, "created" : ISODate("2022-04-06T12:51:59.935Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112735"), "i" : 100330, "username" : "user100330", "age" : 21, "created" : ISODate("2022-04-06T12:51:59.958Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c11279d"), "i" : 100434, "username" : "user100434", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.139Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c1127e5"), "i" : 100506, "username" : "user100506", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.249Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c112842"), "i" : 100599, "username" : "user100599", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.341Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c1128dc"), "i" : 100753, "username" : "user100753", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.492Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c1128e2"), "i" : 100759, "username" : "user100759", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.497Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c1128eb"), "i" : 100768, "username" : "user100768", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.505Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c112921"), "i" : 100822, "username" : "user100822", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.561Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c112a78"), "i" : 101165, "username" : "user101165", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.951Z") }
{ "_id" : ObjectId("624d8cf0e5e7e0a95c112a91"), "i" : 101190, "username" : "user101190", "age" : 21, "created" : ISODate("2022-04-06T12:52:00.978Z") }
{ "_id" : ObjectId("624d8cf1e5e7e0a95c112aae"), "i" : 101219, "username" : "user101219", "age" : 21, "created" : ISODate("2022-04-06T12:52:01.014Z") }
{ "_id" : ObjectId("624d8cf1e5e7e0a95c112b98"), "i" : 101453, "username" : "user101453", "age" : 21, "created" : ISODate("2022-04-06T12:52:01.443Z") }
{ "_id" : ObjectId("624d8cf1e5e7e0a95c112be0"), "i" : 101525, "username" : "user101525", "age" : 21, "created" : ISODate("2022-04-06T12:52:01.536Z") }
{ "_id" : ObjectId("624d8cf1e5e7e0a95c112cd3"), "i" : 101768, "username" : "user101768", "age" : 21, "created" : ISODate("2022-04-06T12:52:01.881Z") }
{ "_id" : ObjectId("624d8c82e5e7e0a95c0fc716"), "i" : 10187, "username" : "user10187", "age" : 21, "created" : ISODate("2022-04-06T12:50:10.409Z") }
{ "_id" : ObjectId("624d8cf2e5e7e0a95c112d51"), "i" : 101894, "username" : "user101894", "age" : 21, "created" : ISODate("2022-04-06T12:52:02.048Z") }
{ "_id" : ObjectId("624d8c82e5e7e0a95c0fc71b"), "i" : 10192, "username" : "user10192", "age" : 21, "created" : ISODate("2022-04-06T12:50:10.510Z") }
Type "it" for more
>
In general , if MongoDB uses an index for a query it will return the resulting documents in index order.
- This is a multivalue query,like the previous one, but this time it has a sort.
>
> db.users.find({"age": {"$gte": 21, "$lte": 30}}).sort({"username":1})
{ "_id" : ObjectId("4b253b067525f35f94b60a31"), "name" : "joe", "age" : 30, "sex" : "male", "location" : "Wisconsin" }
{ "_id" : ObjectId("624d8c77e5e7e0a95c0fa333"), "i" : 1000, "username" : "user1000", "age" : 30, "created" : ISODate("2022-04-06T12:49:59.316Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112604"), "i" : 100025, "username" : "user100025", "age" : 26, "created" : ISODate("2022-04-06T12:51:59.615Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c11260a"), "i" : 100031, "username" : "user100031", "age" : 21, "created" : ISODate("2022-04-06T12:51:59.622Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112616"), "i" : 100043, "username" : "user100043", "age" : 28, "created" : ISODate("2022-04-06T12:51:59.634Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112629"), "i" : 100062, "username" : "user100062", "age" : 22, "created" : ISODate("2022-04-06T12:51:59.654Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112634"), "i" : 100073, "username" : "user100073", "age" : 30, "created" : ISODate("2022-04-06T12:51:59.669Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c11263c"), "i" : 100081, "username" : "user100081", "age" : 30, "created" : ISODate("2022-04-06T12:51:59.679Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112656"), "i" : 100107, "username" : "user100107", "age" : 24, "created" : ISODate("2022-04-06T12:51:59.709Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112667"), "i" : 100124, "username" : "user100124", "age" : 30, "created" : ISODate("2022-04-06T12:51:59.730Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c11267d"), "i" : 100146, "username" : "user100146", "age" : 24, "created" : ISODate("2022-04-06T12:51:59.754Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c11267e"), "i" : 100147, "username" : "user100147", "age" : 29, "created" : ISODate("2022-04-06T12:51:59.755Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112691"), "i" : 100166, "username" : "user100166", "age" : 30, "created" : ISODate("2022-04-06T12:51:59.782Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c112698"), "i" : 100173, "username" : "user100173", "age" : 27, "created" : ISODate("2022-04-06T12:51:59.791Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c1126a2"), "i" : 100183, "username" : "user100183", "age" : 27, "created" : ISODate("2022-04-06T12:51:59.802Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c1126a6"), "i" : 100187, "username" : "user100187", "age" : 25, "created" : ISODate("2022-04-06T12:51:59.807Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c1126c8"), "i" : 100221, "username" : "user100221", "age" : 26, "created" : ISODate("2022-04-06T12:51:59.846Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c1126da"), "i" : 100239, "username" : "user100239", "age" : 21, "created" : ISODate("2022-04-06T12:51:59.864Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c1126e8"), "i" : 100253, "username" : "user100253", "age" : 29, "created" : ISODate("2022-04-06T12:51:59.879Z") }
{ "_id" : ObjectId("624d8cefe5e7e0a95c1126ea"), "i" : 100255, "username" : "user100255", "age" : 28, "created" : ISODate("2022-04-06T12:51:59.881Z") }
Type "it" for more
>
How MongoDB Select an Index
Using Compound Indexes
In the previous sections,we've been using compound indexes,which are indexes with more than one key in them.
To recap, when designing a compound index:
• Keys for equality filters should appear first.
• Keys used for sorting should appear before multivalue fields.
• Keys for multivalue filters should appear last
Choosing key directions
Using covered queries
Implicit indexes
When Not to Index
Types of Indexes
- Unique Indexes
Unique indexes guarantee that each value will appear at most once in the index.
If a key does not exist, the index stores its value as null for that document.
- Compound unique indexes