MAPREDUCE basics - Revision history

Andr3w: Reverted edits by Andr3w (talk) to last revision by 40082593

2016-06-27T11:18:41Z

Reverted edits by Andr3w (talk) to last revision by 40082593

← Older revision		Revision as of 11:18, 27 June 2016
Line 20:		Line 20:


	<div class=q data-lang="~~mongo~~">		<div class=q data-lang="py3">
	<p class=strong>Find the total population of the each continent</p>		<p class=strong>Find the total population of the each continent</p>
	<pre class=def>		<pre class=def>
	db.world.~~mapReduce~~(		from bson.code import Code
	function(){emit(this.continent,this.population);},		temp = db.world.map_reduce(
	function(k,v){return Array.sum(v);},		map=Code("function(){emit(this.continent, this.population)}"),
	{out:{inline:1}}		reduce=Code("""function(key, values){
	)		return Array.sum(values);
			}"""),
			out={"inline":1})
			pp.pprint(temp["results"])
	</pre>		</pre>
	<div class=ans>		<div class=ans>
	db.world.~~mapReduce~~(		from bson.code import Code
	function(){emit(this.continent,this.population);},		temp = db.world.map_reduce(
	function(k,v){return Array.sum(v);},		map=Code("function(){emit(this.continent, this.population)}"),
	{out:{inline:1}}		reduce=Code("""function(key, values){
	)		return Array.sum(values);
			}"""),
			out={"inline":1})
			pp.pprint(temp["results"])
			</div>
	</div>		</div>

Andr3w at 11:16, 27 June 2016

2016-06-27T11:16:47Z

← Older revision		Revision as of 11:16, 27 June 2016
Line 20:		Line 20:


	<div class=q data-lang="~~py3~~">		<div class=q data-lang="mongo">
	<p class=strong>Find the total population of the each continent</p>		<p class=strong>Find the total population of the each continent</p>
	<pre class=def>		<pre class=def>
	~~from bson.code import Code~~		db.world.mapReduce(
	~~temp =~~ db.world.~~map_reduce~~(		function(){emit(this.continent,this.population);},
	~~map=Code("~~function(){emit(this.continent, this.population)}"),		function(k,v){return Array.sum(v);},
	~~reduce=Code("""~~function(~~key~~, ~~values~~){		{out:{inline:1}}
	return Array.sum(~~values~~);		)
	}~~""")~~,
	out={"inline":1})
	~~pp.pprint(temp["results"])~~
	</pre>		</pre>
	<div class=ans>		<div class=ans>
	~~from bson.code import Code~~		db.world.mapReduce(
	~~temp =~~ db.world.~~map_reduce~~(		function(){emit(this.continent,this.population);},
	~~map=Code("~~function(){emit(this.continent, this.population)}"),		function(k,v){return Array.sum(v);},
	~~reduce=Code("""~~function(~~key~~, ~~values~~){		{out:{inline:1}}
	return Array.sum(~~values~~);		)
	}~~""")~~,
	out={"inline":1})
	~~pp.pprint(temp["results"]~~)
	~~</div>~~
	</div>		</div>

40082593 at 13:47, 30 July 2015

2015-07-30T13:47:25Z

@@ Line 156: / Line 156: @@
 </div>
 </div>
 <div class=q data-lang="py3">
@@ Line 183: / Line 182: @@
           return val;
      }"""),
      out={"inline":1})
 pp.pprint(temp["results"])
 </div>
 </div>

40082593 at 13:05, 30 July 2015

2015-07-30T13:05:55Z

← Older revision		Revision as of 13:05, 30 July 2015
Line 136:		Line 136:

	<div class=q data-lang="py3">		<div class=q data-lang="py3">
	<p class=strong>~~Without using <code>query</code> return the~~ name ~~and capital for any country~~ or capital that starts with an 'L'</p>		<p class=strong>Return country name or capital city that starts with a letter 'M'</p>
	<pre class=def>		<pre class=def>
	</pre>		</pre>
Line 142:		Line 142:
	from bson.code import Code		from bson.code import Code
	temp = db.world.map_reduce(		temp = db.world.map_reduce(
	map=Code("""function(){ if((this.name).startsWith('L')\|(this.capital).startsWith('L'))emit(this.~~name~~, ~~this.capital~~)}"""),		map=Code("""function(){
			if((this.name).startsWith('M'))
			emit(this.name,null);
			if((this.capital).startsWith('M'))
			emit(this.capital,null);
			}"""),
	reduce=Code("""function(key, values){		reduce=Code("""function(key, values){
	return values;		return values;
	}"""),		}"""),

	out={"inline":1})		out={"inline":1})
	pp.pprint(temp["results"])		pp.pprint(temp["results"])
Line 151:		Line 157:
	</div>		</div>


	~~<div class=q data-lang="py3">~~
	~~<p class=strong>Count number of countries by first letter</p>~~
	~~<pre class=def>~~
	~~</pre>~~
	~~<div class=ans>~~
	~~from bson.code import Code~~
	~~temp = db.world.map_reduce(~~
	~~map=Code("""function(){ emit((this.name).substring(0,1), 1)}"""),~~
	~~reduce=Code("""function(key, values){~~
	~~return Array.sum(values);~~
	~~}"""),~~
	~~out={"inline":1})~~
	~~pp.pprint(temp["results"])~~
	~~</div>~~
	~~</div>~~

	<div class=q data-lang="py3">		<div class=q data-lang="py3">

40082593 at 12:37, 30 July 2015

2015-07-30T12:37:10Z

@@ Line 11: / Line 11: @@
 #PRETTY
 import pprint
-pp = pprint.PrettyPrinter(indent=4)
+pp = pprint.PrettyPrinter(indent=4, width=160)
 </pre>
@@ Line 162: / Line 162: @@
      reduce=Code("""function(key, values){
         return Array.sum(values);
      }"""),
      out={"inline":1})

40082593 at 12:17, 30 July 2015

2015-07-30T12:17:41Z

← Older revision		Revision as of 12:17, 30 July 2015
Line 18:		Line 18:
	[[MapReduce]] examples are available.		[[MapReduce]] examples are available.
	<div class='extra_space' style='width:1em; height:6em;'></div>		<div class='extra_space' style='width:1em; height:6em;'></div>


	<div class=q data-lang="py3">		<div class=q data-lang="py3">
Line 54:		Line 55:
	reduce=Code("""function(key, values){		reduce=Code("""function(key, values){
	return Math.round(Array.sum(values)/1000000)*1000000;		return Math.round(Array.sum(values)/1000000)*1000000;
			}"""),
			out={"inline":1})
			pp.pprint(temp["results"])
			</div>
			</div>

			<div class=q data-lang="py3">
			<p class=strong>Count number of countries by first letter</p>
			<pre class=def>
			</pre>
			<div class=ans>
			from bson.code import Code
			temp = db.world.map_reduce(
			map=Code("""function(){ emit((this.name).substring(0,1), 1)}"""),
			reduce=Code("""function(key, values){
			return Array.sum(values);
	}"""),		}"""),
	out={"inline":1})		out={"inline":1})
Line 104:		Line 121:
	from bson.code import Code		from bson.code import Code
	temp = db.world.map_reduce(		temp = db.world.map_reduce(
	map=Code("function(){emit(this.continent, {~~min~~:this.name,~~max~~:this.name})}"),		map=Code("function(){emit(this.continent, {first:this.name,last:this.name})}"),
	reduce=Code("""function(key, values){		reduce=Code("""function(key, values){
	var ret = {~~min~~:'ZZZ',~~max~~:'AAA'};		var ret = {first:'ZZZ',last:'AAA'};
	for(var i=0;i<values.length;i++){		for(var i=0;i<values.length;i++){
	if (ret.~~min~~>values[i].~~min~~) ret.~~min~~=values[i].~~min~~;		if (ret.first>values[i].first) ret.first=values[i].first;
	if (ret.~~max~~<values[i].~~max~~) ret.~~max~~=values[i].~~max~~;		if (ret.last<values[i].last) ret.last=values[i].last;
	}		}
	return ret;		return ret;
Line 119:		Line 136:

	<div class=q data-lang="py3">		<div class=q data-lang="py3">
	<p class=strong>Without using query return the name and capital for any country or capital that starts with an 'L'</p>		<p class=strong>Without using <code>query</code> return the name and capital for any country or capital that starts with an 'L'</p>
	<pre class=def>		<pre class=def>
	</pre>		</pre>
Line 128:		Line 145:
	reduce=Code("""function(key, values){		reduce=Code("""function(key, values){
	return values;		return values;
			}"""),
			out={"inline":1})
			pp.pprint(temp["results"])
			</div>
			</div>


			<div class=q data-lang="py3">
			<p class=strong>Count number of countries by first letter</p>
			<pre class=def>
			</pre>
			<div class=ans>
			from bson.code import Code
			temp = db.world.map_reduce(
			map=Code("""function(){ emit((this.name).substring(0,1), 1)}"""),
			reduce=Code("""function(key, values){
			return Array.sum(values);
	}"""),		}"""),
	out={"inline":1})		out={"inline":1})

40082593 at 11:24, 30 July 2015

2015-07-30T11:24:59Z

← Older revision		Revision as of 11:24, 30 July 2015
Line 44:		Line 44:

	<div class=q data-lang="py3">		<div class=q data-lang="py3">
	<p class=strong>~~Show~~ the ~~number~~ of ~~countries on each continent~~</p>		<p class=strong>Use the previous answer to find the population of the world to the nearest million</p>
			<div class=hint title="How to round to the nearest million">Use the JavaScript round function : Math.round(population/1000000)*1000000 </div>
	<pre class=def>		<pre class=def>
			</pre>
			<div class=ans>
	from bson.code import Code		from bson.code import Code
	temp = db.world.map_reduce(		temp = db.world.map_reduce(
	map=Code("function(){emit(this.~~continent, 1~~)}"),		map=Code("function(){emit('World Population in Millions', this.population)}"),
	reduce=Code("""function(key, values){		reduce=Code("""function(key, values){
	return Array.sum(values);		return Math.round(Array.sum(values)/1000000)*1000000;
	}"""),		}"""),
	out={"inline":1})		out={"inline":1})
	pp.pprint(temp["results"])		pp.pprint(temp["results"])
			</div>
			</div>

			<div class=q data-lang="py3">
			<p class=strong>Show the number of countries on each continent</p>
			<pre class=def>
	</pre>		</pre>
	<div class=ans>		<div class=ans>
Line 68:		Line 77:

	<div class=q data-lang="py3">		<div class=q data-lang="py3">
	<p class=strong>Show the smallest 3 countries (ignore areas of 0 or None)</p>		<p class=strong>Show the smallest 3 countries name and area (ignore areas of 0 or None)</p>
	<pre class=def>		<pre class=def>
	</pre>		</pre>
Line 110:		Line 119:

	<div class=q data-lang="py3">		<div class=q data-lang="py3">
	<p class=strong>~~Return~~ the ~~first~~ and ~~last~~ country ~~based on name order for each continent~~</p>		<p class=strong>Without using query return the name and capital for any country or capital that starts with an 'L'</p>
	<pre class=def>		<pre class=def>
	</pre>		</pre>
Line 116:		Line 125:
	from bson.code import Code		from bson.code import Code
	temp = db.world.map_reduce(		temp = db.world.map_reduce(
	map=Code("function(){~~emit~~(this.~~continent, {min:~~this.name,~~max:~~this.~~name}~~)}"),		map=Code("""function(){ if((this.name).startsWith('L')\|(this.capital).startsWith('L'))emit(this.name, this.capital)}"""),
	reduce=Code("""function(key, values){		reduce=Code("""function(key, values){
	~~var ret = {min:'ZZZ',max:'AAA'};~~		return values;
	~~for(var i=0;i<values.length;i++){~~
	~~if (ret.min>values[i].min) ret.min=values[i].min;~~
	~~if (ret.max<~~values~~[i].max) ret.max=values[i].max;~~
	}
	~~return ret~~;
	}"""),		}"""),
	out={"inline":1})		out={"inline":1})

40082593 at 10:40, 30 July 2015

2015-07-30T10:40:14Z

← Older revision		Revision as of 10:40, 30 July 2015
Line 20:		Line 20:

	<div class=q data-lang="py3">		<div class=q data-lang="py3">
	<p class=strong>~~Show a list~~ of ~~countries found in North America~~</p>		<p class=strong>Find the total population of the each continent</p>
	<pre class=def>		<pre class=def>
	from bson.code import Code		from bson.code import Code
	temp = db.world.map_reduce(		temp = db.world.map_reduce(
	~~query={"continent":"Africa"},~~		map=Code("function(){emit(this.continent, this.population)}"),
	map=Code("function(){emit(this.continent, this.~~name~~)}"),		reduce=Code("""function(key, values){
	reduce=Code("function(key, values){return ~~values~~.~~toString~~()}"),		return Array.sum(values);
	out={"inline":1}		}"""),
	)		out={"inline":1})
			pp.pprint(temp["results"])
	pp.pprint(
	temp["results"]
	)
	</pre>		</pre>
	<div class=ans>		<div class=ans>
	from bson.code import Code		from bson.code import Code
	temp = db.world.map_reduce(~~query={"continent":"North America"},~~map=Code("function(){emit(this.continent, this.~~name~~)}"), reduce=Code("function(key, values){return ~~values~~.~~toString~~()}"),out={"inline":1},)		temp = db.world.map_reduce(
			map=Code("function(){emit(this.continent, this.population)}"),
			reduce=Code("""function(key, values){
			return Array.sum(values);
			}"""),
			out={"inline":1})
	pp.pprint(temp["results"])		pp.pprint(temp["results"])
	</div>		</div>
Line 44:		Line 46:
	<p class=strong>Show the number of countries on each continent</p>		<p class=strong>Show the number of countries on each continent</p>
	<pre class=def>		<pre class=def>
			from bson.code import Code
			temp = db.world.map_reduce(
			map=Code("function(){emit(this.continent, 1)}"),
			reduce=Code("""function(key, values){
			return Array.sum(values);
			}"""),
			out={"inline":1})
			pp.pprint(temp["results"])
	</pre>		</pre>
	<div class=ans>		<div class=ans>
Line 75:		Line 85:
	temp["results"]		temp["results"]
	)		)
			</div>
			</div>

			<div class=q data-lang="py3">
			<p class=strong>Return the first and last country based on name order for each continent</p>
			<pre class=def>
			</pre>
			<div class=ans>
			from bson.code import Code
			temp = db.world.map_reduce(
			map=Code("function(){emit(this.continent, {min:this.name,max:this.name})}"),
			reduce=Code("""function(key, values){
			var ret = {min:'ZZZ',max:'AAA'};
			for(var i=0;i<values.length;i++){
			if (ret.min>values[i].min) ret.min=values[i].min;
			if (ret.max<values[i].max) ret.max=values[i].max;
			}
			return ret;
			}"""),
			out={"inline":1})
			pp.pprint(temp["results"])
	</div>		</div>
	</div>		</div>

40082593 at 09:48, 30 July 2015

2015-07-30T09:48:59Z

@@ Line 47: / Line 47: @@
 <div class=ans>
 from bson.code import Code
-temp = db.world.map_reduce(map=Code("function(){emit(this.continent, this.name)}"), reduce=Code("function(key, values){return values.length}"),out={"inline":1})
+temp = db.world.map_reduce(
 pp.pprint(temp["results"])
 </div>
@@ Line 70: / Line 75: @@
     temp["results"]
+)
 </div>
 </div>

40082593 at 15:11, 29 July 2015

2015-07-29T15:11:10Z

@@ Line 13: / Line 13: @@
 pp = pprint.PrettyPrinter(indent=4)
 </pre>
 <div class=q data-lang="py3">
@@ Line 44: / Line 49: @@
 temp = db.world.map_reduce(map=Code("function(){emit(this.continent, this.name)}"), reduce=Code("function(key, values){return values.length}"),out={"inline":1})
 pp.pprint(temp["results"])
 </div>
 </div>