用Python写map,reduce

用Python写map,reduce

#!/usr/bin/env python
import sys
for line in sys.stdin:
    line= line.strip()
    words= line.split()
    for word in words:
        print'%s\t%s' % (word,1)
--map.py
#!/usr/bin/env python
 
from operator import itemgetter
import sys
word2count = {}
for line in sys.stdin:
    line = line.strip()
    word, count =line.split('\t', 1)
    try:
        count =int(count)
        word2count[word]=word2count.get(word,0)+count
    except ValueError:
        pass
sorted_word2count = sorted(word2count.items(),key=itemgetter(0))
for word,count in sorted_word2count:
    print'%s\t%s' % (word, count)

--
测试：
echo "foo fooquux labs foo bar quux" | ./mapper.py | sort | ./reducer.py
提交HADOOP集群运行：

      hadoop jar hadoop-1.2.1/contrib/streaming/hadoop-streaming-1.2.1.jar -input input -output output-streaming-python  -mapper /home/mapper.py -reducer /home/reducer.py