Cookies help us deliver our services. By using our services, you agree to our use of cookies. More information

Difference between revisions of "MAPREDUCE Tutorial"

From NoSQLZoo
Jump to: navigation, search
Line 13: Line 13:
 
<div class="q" data-lang="mongo">
 
<div class="q" data-lang="mongo">
 
<p class="strong">Use a map function to get the names of countries in Europe.</p>
 
<p class="strong">Use a map function to get the names of countries in Europe.</p>
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 25: Line 25:
 
   {out:{inline:1}}   
 
   {out:{inline:1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){if(this.continent=="Europe"){emit(this.name,null)}},function(_,v){return v},{out:{inline:1}})
  function(){if (this.continent=="Europe"){emit(this.name,null);}},
+
</nowiki></pre>
  function(k, v){ return v; },
 
  {out:{inline:1}}
 
)
 
</pre>
 
 
</div>
 
</div>
  
Line 38: Line 34:
 
<p class="strong">Use the previous answer to find the population of the world to the nearest million.</p>
 
<p class="strong">Use the previous answer to find the population of the world to the nearest million.</p>
 
<div class="hint" title="How to round to the nearest million">Use the JavaScript round function : Math.round(population/1000000)*1000000 </div>
 
<div class="hint" title="How to round to the nearest million">Use the JavaScript round function : Math.round(population/1000000)*1000000 </div>
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 48: Line 44:
 
   {out:{inline:1}}   
 
   {out:{inline:1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){emit('World Population in Millions',this.population)},function(_,values){return Math.round(Array.sum(values)/1000000)*1000000},{out:{inline:1}})
    function(){emit('World Population in Millions', this.population)},
+
</nowiki></pre>
    function(key, values){
 
      return Math.round(Array.sum(values)/1000000)*1000000;
 
    },
 
    {out: {inline :1}}
 
)
 
</pre>
 
 
</div>
 
</div>
  
 
<div class="q" data-lang="mongo">
 
<div class="q" data-lang="mongo">
 
<p class="strong">For each letter, determine how many country names begin with that letter.</p>
 
<p class="strong">For each letter, determine how many country names begin with that letter.</p>
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 70: Line 60:
 
     return null;  
 
     return null;  
 
   },
 
   },
   {out:{inline:1}}   
+
   {out:{inline: 1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){emit((this.name).substring(0,1),1)},function(_,values){return Array.sum(values)},{out:{inline:1}})
    function(){ emit((this.name).substring(0,1), 1)},
+
</nowiki></pre>
    function(key, values){
 
      return Array.sum(values);
 
    },
 
    {out: {inline:1}}
 
)
 
</pre>
 
 
</div>
 
</div>
  
 
<div class="q" data-lang="mongo">
 
<div class="q" data-lang="mongo">
 
<p class="strong">Show the number of countries in each continent.</p>
 
<p class="strong">Show the number of countries in each continent.</p>
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 96: Line 80:
 
   {out:{inline:1}}   
 
   {out:{inline:1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){emit(this.continent,1)},function(_,v){return Array.sum(v)},{out:{inline:1}})
    function(){emit(this.continent, 1)},
+
</nowiki></pre>
    function(key, values){
 
      return Array.sum(values);
 
    },
 
    {out: {inline :1}}
 
)
 
</pre>
 
 
</div>
 
</div>
  
 
<div class="q" data-lang="mongo">
 
<div class="q" data-lang="mongo">
 
<p class="strong">Show the name and area of the smallest 3 countries by area. Ignore records where the area is 0 or null.</p>
 
<p class="strong">Show the name and area of the smallest 3 countries by area. Ignore records where the area is 0 or null.</p>
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 120: Line 98:
 
   {out:{inline:1}}   
 
   {out:{inline:1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){emit(this.name,this.area)},function(_,v){return v},{limit:3,out:{inline:1},query:{"$and":[{"area":{"$ne":null}},{"area":{"$ne":0}}]},sort:{"area":1}})
        function(){emit(this.name, this.area);},  
+
</nowiki></pre>
        function(k, v){ return v;},
 
        {
 
            limit: 3,
 
            out: {inline:1},
 
            query: {"$and": [{"area": {"$ne": null}}, {"area": {"$ne": 0}}]},
 
            sort: {"area": 1}
 
        }
 
)
 
</pre>
 
 
</div>
 
</div>
  
 
<div class="q" data-lang="mongo">
 
<div class="q" data-lang="mongo">
 
<p class="strong">For each continent, find the first and last country alphabetically.</p>
 
<p class="strong">For each continent, find the first and last country alphabetically.</p>
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 147: Line 116:
 
   {out:{inline:1}}   
 
   {out:{inline:1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){emit(this.continent,{first:this.name,last:this.name})},function(_,v){let r={first:'ZZZ',last:'AAA'};for(let i=0;i<v.length;i++){if(r.first>v[i].first){r.first=v[i].first}
    function(){emit(this.continent, {first:this.name, last:this.name});},
+
if(r.last<v[i].last){r.last=v[i].last}}
    function(key, values){
+
return r},{out:{inline:1}})
      let ret = {first:'ZZZ', last:'AAA'};
+
</nowiki></pre>
      for(let i=0; i < values.length; i++){
 
        if (ret.first > values[i].first) { ret.first = values[i].first; }
 
        if (ret.last < values[i].last) { ret.last = values[i].last; }
 
      }
 
      return ret;
 
    },
 
    {out: {inline:1}}
 
)
 
</pre>
 
 
</div>
 
</div>
 
<div class="q" data-lang="mongo">
 
<div class="q" data-lang="mongo">
 
<p class="strong">Return country names or capital cities that start with a letter 'M' as keys, use null as the value.</p>
 
<p class="strong">Return country names or capital cities that start with a letter 'M' as keys, use null as the value.</p>
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 175: Line 135:
 
   {out:{inline:1}}   
 
   {out:{inline:1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){if((this.name).startsWith('M')){emit(this.name,null)}
    function(){  
+
if((this.capital).startsWith('M')){emit(this.capital,null)}},function(_,v){return v},{out:{inline:1}})
        if((this.name).startsWith('M')) {emit(this.name,null); }
+
</nowiki></pre>
        if((this.capital).startsWith('M')) {emit(this.capital,null); }
 
    },
 
    function(key, values){
 
      return values;
 
    },  
 
    {out: {inline :1}}
 
)
 
</pre>
 
 
</div>
 
</div>
  
Line 202: Line 154:
 
},
 
},
 
</pre>
 
</pre>
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 212: Line 164:
 
   {out:{inline:1}}   
 
   {out:{inline:1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){if(this.capital){emit((this.capital).substring(0,1),{first:this.capital,last:this.capital})}},function(_,v){let r={first:'ZZZ',last:'AAA',count:0};for(let i=0;i<v.length;i++){if(r.first>v[i].first){r.first=v[i].first};if(r.last<v[i].last){r.last=v[i].last};r.count+=1}
    function(){ if(this.capital)emit((this.capital).substring(0,1), {first:this.capital, last:this.capital})},
+
return r},{finalize:function(_,v){if(!v.count){v.count=1}
    function(key, values){
+
return v},out:{inline:1}})
      let ret = {first:'ZZZ',last:'AAA', count:0};
+
</nowiki></pre>
      for(let i=0; i < values.length; i++){
 
        if (ret.first>values[i].first) { ret.first = values[i].first };
 
        if (ret.last<values[i].last) { ret.last = values[i].last };
 
        ret.count += 1;
 
      }
 
      return ret;
 
    },
 
    {
 
        finalize: function(key, val){
 
            if(!val.count){
 
              val.count = 1;
 
            }
 
            return val;
 
        },
 
        out: {inline: 1}
 
    }
 
)
 
</pre>
 
 
</div>
 
</div>
  
Line 247: Line 181:
 
   10000000 to 15000000
 
   10000000 to 15000000
 
   More than 15000000   
 
   More than 15000000   
<pre class="def">
+
<pre class="def"><nowiki>
 
db.world.mapReduce(
 
db.world.mapReduce(
 
   function(){
 
   function(){
Line 257: Line 191:
 
   {out:{inline:1}}   
 
   {out:{inline:1}}   
 
)
 
)
</pre>
+
</nowiki></pre>
<pre class="ans">
+
<pre class="ans"><nowiki>
db.world.mapReduce(
+
db.world.mapReduce(function(){var p=this.population;switch(!0){case p<1000000:emit("0 TO 1000000",1);break;case p<2000000:emit("1000000 TO 2000000",1);break;case p<3000000:emit("2000000 TO 3000000",1);break;case p<5000000:emit("3000000 TO 5000000",1);break;case p<10000000:emit("5000000 TO 10000000",1);break;case p<15000000:emit("10000000 TO 15000000",1);break
    function(){
+
case p>15000000:emit("MORE THAN 15000000",1);break}},function(_,v){return Array.sum(v)},{out:{inline:1}})
        var pop = this.population;
+
</nowiki></pre>
        switch(true){
 
            case pop<1000000:
 
              emit("0 TO 1000000", 1);
 
              break;
 
            case pop<2000000:
 
              emit("1000000 TO 2000000", 1);
 
              break;
 
            case pop<3000000:
 
              emit("2000000 TO 3000000", 1);
 
              break;
 
            case pop<5000000:
 
              emit("3000000 TO 5000000", 1);
 
              break;
 
            case pop<10000000:
 
              emit("5000000 TO 10000000", 1);
 
              break;
 
            case pop<15000000:
 
              emit("10000000 TO 15000000", 1);
 
              break
 
            case pop>15000000:
 
              emit("MORE THAN 15000000", 1);
 
              break;
 
        }
 
    },
 
    function(key, values){ return Array.sum(values); },
 
    {out: {inline: 1}}  
 
)
 
</pre>
 
 
</div>
 
</div>

Revision as of 14:21, 15 June 2018

MapReduce the basics

This tutorial introduces the MapReduce command.

MapReduce examples are available.

The MapReduce function allows to functions to be applied over a collection. The two functions map and reduce are important because:
  • Together they can solve a huge range of common database queries.
  • They can be distributed effectively; you can share the work load among a number of processors and get your answer faster.

It is recommended to have some familiarity with JavaScript before attempting the following questions: JavaScript | MDN

Map

The map function can be used to filter and transform the data:

Use a map function to get the names of countries in Europe.

db.world.mapReduce(
  function(){
    if (this.continent === "Europe"){
      emit(this.name, null);
    }
  },
  function(k, v){ 
    return v; 
  },
  {out:{inline:1}}  
)
db.world.mapReduce(function(){if(this.continent=="Europe"){emit(this.name,null)}},function(_,v){return v},{out:{inline:1}})

Use the previous answer to find the population of the world to the nearest million.

Use the JavaScript round function : Math.round(population/1000000)*1000000
db.world.mapReduce(
  function(){
    emit(null, null);
  },
  function(k, v){ 
    return null; 
  },
  {out:{inline:1}}  
)
db.world.mapReduce(function(){emit('World Population in Millions',this.population)},function(_,values){return Math.round(Array.sum(values)/1000000)*1000000},{out:{inline:1}})

For each letter, determine how many country names begin with that letter.

db.world.mapReduce(
  function(){
    emit(null, null);
  },
  function(k, v){ 
    return null; 
  },
  {out:{inline: 1}}  
)
db.world.mapReduce(function(){emit((this.name).substring(0,1),1)},function(_,values){return Array.sum(values)},{out:{inline:1}})

Show the number of countries in each continent.

db.world.mapReduce(
  function(){
    emit(null, null);
  },
  function(k, v){ 
    return null; 
  },
  {out:{inline:1}}  
)
db.world.mapReduce(function(){emit(this.continent,1)},function(_,v){return Array.sum(v)},{out:{inline:1}})

Show the name and area of the smallest 3 countries by area. Ignore records where the area is 0 or null.

db.world.mapReduce(
  function(){
    emit(null, null);
  },
  function(k, v){ 
    return null; 
  },
  {out:{inline:1}}  
)
db.world.mapReduce(function(){emit(this.name,this.area)},function(_,v){return v},{limit:3,out:{inline:1},query:{"$and":[{"area":{"$ne":null}},{"area":{"$ne":0}}]},sort:{"area":1}})

For each continent, find the first and last country alphabetically.

db.world.mapReduce(
  function(){
    emit(null, null);
  },
  function(k, v){ 
    return null; 
  },
  {out:{inline:1}}  
)
db.world.mapReduce(function(){emit(this.continent,{first:this.name,last:this.name})},function(_,v){let r={first:'ZZZ',last:'AAA'};for(let i=0;i<v.length;i++){if(r.first>v[i].first){r.first=v[i].first}
if(r.last<v[i].last){r.last=v[i].last}}
return r},{out:{inline:1}})

Return country names or capital cities that start with a letter 'M' as keys, use null as the value.

db.world.mapReduce(
  function(){
    emit(null, null);
  },
  function(k, v){ 
    return null; 
  },
  {out:{inline:1}}  
)
db.world.mapReduce(function(){if((this.name).startsWith('M')){emit(this.name,null)}
if((this.capital).startsWith('M')){emit(this.capital,null)}},function(_,v){return v},{out:{inline:1}})

For each letter in the alphabet, find the amount of cities and the cities that come first and last alphabetically. e.g.

{
	"_id" : "B",
	"value" : {
		"first" : "Baghdad",
		"last" : "Bujumbura",
		"count" : 25
	}
},
db.world.mapReduce(
  function(){
    emit(null, null);
  },
  function(k, v){ 
    return null; 
  },
  {out:{inline:1}}  
)
db.world.mapReduce(function(){if(this.capital){emit((this.capital).substring(0,1),{first:this.capital,last:this.capital})}},function(_,v){let r={first:'ZZZ',last:'AAA',count:0};for(let i=0;i<v.length;i++){if(r.first>v[i].first){r.first=v[i].first};if(r.last<v[i].last){r.last=v[i].last};r.count+=1}
return r},{finalize:function(_,v){if(!v.count){v.count=1}
return v},out:{inline:1}})

Show country count for countries in the following ranges:

 0 to 1000000
 1000000 to 2000000
 2000000 to 3000000
 3000000 to 5000000
 5000000 to 10000000
 10000000 to 15000000
 More than 15000000  
db.world.mapReduce(
  function(){
    emit(null, null);
  },
  function(k, v){ 
    return null; 
  },
  {out:{inline:1}}  
)
db.world.mapReduce(function(){var p=this.population;switch(!0){case p<1000000:emit("0 TO 1000000",1);break;case p<2000000:emit("1000000 TO 2000000",1);break;case p<3000000:emit("2000000 TO 3000000",1);break;case p<5000000:emit("3000000 TO 5000000",1);break;case p<10000000:emit("5000000 TO 10000000",1);break;case p<15000000:emit("10000000 TO 15000000",1);break
case p>15000000:emit("MORE THAN 15000000",1);break}},function(_,v){return Array.sum(v)},{out:{inline:1}})