Difference between revisions of "MapReduce"

#ENCODING
import io
import sys
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-16')
#MONGO
from pymongo import MongoClient
client = MongoClient()
client.progzoo.authenticate('scott','tiger')
db = client['progzoo']
#PRETTY
import pprint
pp = pprint.PrettyPrinter(indent=4)

db.<collection>.map_reduce(
    map=<function>,
    reduce=<function>,
    out=<collection>
)

In this example we will be returning the population of all the continents.

emit(k,v) lets us pick the fields we want to turn into tuples, where k is the key and v is the value. Our keys will be the continents and our values will be the population
In our reduce we sum all the values associated with each key. Finally we specify that we want the "out" part of the MapReduce to be inline rather than a collection, allowing us to print it to screen.

from bson.code import Code
pp.pprint(
    db.world.map_reduce(
        map=Code("function(){emit(this.continent, this.population)}"), 
        reduce=Code("function(key, values){"
                    "    var total = 0;"
                    "    for (var i = 0; i < values.length; i++){"
                    "        total += values[i];"
                    "    }"
                    "    return total;"
                    "}"),
        out={"inline":1},
    )
)

This is all very good, but what if we only want the results of a MapReduce? Whilst we would normally write our results to another collection, for these examples we will store our results as a dict, then access the results through Python rather than PyMongo.

Alternatively, if you want more information about the timing you can use verbose=1

Find the average population of each country by continent

from bson.code import Code
temp = db.world.map_reduce(
        map=Code("function(){emit(this.continent, this.population)}"), 
        reduce=Code("function(key, values){"
                  "    var total = 0;"
                  "    for (var i = 0; i < values.length; i++){"
                  "        total += values[i];"
                  "    }"
                  "    return total/values.length;"
                  "}"),
        out={"inline":1},
)

pp.pprint(list(
   temp["results"]
))

This also allows us to do things like sorting, again for this we use Python, and not PyMongo.

Find the average population of each country by continent then sort by value in descending order.

from bson.code import Code
temp = db.world.map_reduce(
        map=Code("function(){emit(this.continent, this.population)}"), 
        reduce=Code("function(key, values){"
                  "    var total = 0;"
                  "    for (var i = 0; i < values.length; i++){"
                  "        total += values[i];"
                  "    }"
                  "    return total/values.length;"
                  "}"),
        out={"inline":1},
)

import operator
pp.pprint(list(
   sorted(temp["results"],key=operator.itemgetter('value'), reverse=True)
))

Until now we've been working on all the data, let's apply a mapreduce to a more specific set of elements by using query

Find the GDP for each continent, but only include data from countries that start with the letter A or B.

from bson.code import Code
temp = db.world.map_reduce(
        query={"name": {"$regex":"^(A|B)"}},
        map=Code("function(){emit(this.continent, this.gdp)}"), 
        reduce=Code("function(key, values){"
                  "    var total = 0;"
                  "    for (var i = 0; i < values.length; i++){"
                  "        total += values[i];"
                  "    }"
                  "    return total/values.length;"
                  "}"),
        out={"inline":1},
)

import operator
pp.pprint(list(
   temp["results"]
))

Another MapReduce feature is scope, which takes in a document:{} and lets you create global variables.

It's syntax is: scope={}.

Using scope, list all the countries with a higher population than Mexico.

mexico_data = db.world.find_one({"name":"Mexico"})
pp.pprint(mexico_data)

from bson.code import Code
temp = db.world.map_reduce(
        scope={"MEXICO":mexico_data},
        map=Code("function(){"
                 "    if (this.population > MEXICO.population) emit(this.name, this.population)"
                 "}"), 
        reduce=Code("function(key, values){"
                  "    var total = 0;"
                  "    for (var i = 0; i < values.length; i++){"
                  "        total += values[i];"
                  "    }"
                  "    return total/values.length;"
                  "}"),
        out={"inline":1},
)
import operator
pp.pprint(list(
   sorted(temp["results"],key=operator.itemgetter('value'), reverse=True)
))