and
a collection of daemons for clustering systems
provides redundant storage and parallel computation
handles petabytes of data, scales linearly
not suitable for high velocity data (runs batch jobs)
it's not just for java developers
one script each for the map and reduce phases
read key/value pairs from stdin, write to stdout
operate on a portion of the input files
don't share state, can't see the whole set of input records
write intermediate key-value pairs for reducers
hadoop jar /usr/lib/hadoop-0.20-mapreduce/contrib/streaming/hadoop-streaming-2.0.0-mr1-cdh4.2.0.jar -input /user/cloudera/in -output /user/cloudera/out7/ -mapper map.rb -reducer reduce.rb -file map.rb -file reduce.rb #!/usr/bin/ruby
ARGF.each do |line|
line.split(' ').each do |word|
puts "#{word}\t1"
end
end #!/usr/bin/ruby count=0 ARGF.each do |line| w = line.split("\t")[0] if w == word count +=1else if count != 0 puts "#{word}\t#{count+1}" end word = w count = 0 end end
# Create the tableCREATE TABLE soccer ( sid int, ts bigint, x double, y double, ... )ROW FORMAT DELIMITED FIELDS TERMINATED BY ',';# Load the dataLOAD DATA LOCAL INPATH 'full-game' INTO TABLE soccer;
get '/data' do
content_type 'application/json'
data=[]
Impala.connect(IMPALA_SERVER, 21000) do |conn|
data = conn.query("select (round(-1*x/2624)*25)+563 as y, (round(-1*y/3400)*39)+472 as x, count(*)/200 as val from soccer_part WHERE sid =#{params[:sid]} AND ts <= #{params[:max]} AND ts >= #{params[:min]} AND x > 0 group by x,y")
end
return JSON.dump data
end