Question

When i try to run a word-count prg in mapreduce using oozie.. It just reads the input records and displays it. I guess its not even invoking my mapper and reducer classes.Since i am using the new API, have included the new-api property tag also in workflow.xml.

Map-reduce snippet:

public class WordCount {

  public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> {
     private final static IntWritable one = new IntWritable(1);
     private Text word = new Text();

     public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
         String line = value.toString();
         StringTokenizer tokenizer = new StringTokenizer(line);
         while (tokenizer.hasMoreTokens()) {
             word.set(tokenizer.nextToken());
             context.write(word, one);
         }
 }

}

  public static class Reduce extends Reducer<Text,IntWritable,Text,IntWritable> {

     public void reduce(Text key, Iterable<IntWritable> values, Context context) 
       throws IOException, InterruptedException {
         int sum = 0;
         for (IntWritable val : values) {
             sum += val.get();
         }
             context.write(key, new IntWritable(sum));
     }

}

my workflow.xml:

    <?xml version="1.0" encoding="UTF-8"?>
    <workflow-app xmlns='uri:oozie:workflow:0.1' name="wordcount">
    <start to="wc-node" />
    <action name="wc-node">
    <map-reduce>
        <job-tracker>${jobTracker}</job-tracker>
        <name-node>${nameNode}</name-node>
        <prepare>
            <delete path="${nameNode}/user/${wf:user()}/${wordcountRoot}/output-    data/${outputDir}"/>
        </prepare>

        <configuration>

            <property>
                <name>mapred.mapper.new-api</name>
                <value>true</value>
            </property>

            <property>
                <name>mapred.reducer.new-api</name>
                <value>true</value>
                 </property>

            <property>
                <name>mapreduce.map.class</name>
                <value>WordCount.Map</value>
            </property>

            <property>
                <name>mapreduce.reduce.class</name>
                <value>WordCount.Reduce</value>
            </property>

            <property>
              <name>mapred.output.key.class</name>
              <value>org.apache.hadoop.io.Text</value>
            </property>

            <property>
               <name>mapred.output.value.class</name>
               <value>org.apache.hadoop.io.IntWritable</value>
            </property>

            <property>
                <name>mapred.map.tasks</name>
                <value>1</value>
            </property>

            <property>
                <name>mapred.input.dir</name>
                <value>/user/${wf:user()}/${wordcountRoot}/input-data</value>
            </property>
            <property>
                <name>mapred.output.dir</name>
                <value>/user/${wf:user()}/${wordcountRoot}/output-data/${outputDir}</value>
            </property>

            <property>
                <name>mapred.job.queue.name</name>
                <value>${queueName}</value>
            </property>

            <property>
             <name>mapreduce.job.acl-view-job</name>
             <value>*</value>
            </property>

            <property>
               <name>oozie.launcher.mapreduce.job.acl-view-job</name>
               <value>*</value>
            </property>

         </configuration>

    </map-reduce>

    <ok to="end" />
    <error to="fail" />
</action>

<kill name="fail">
    <message>Map/Reduce failed</message>
</kill>
<end name="end" />

I referred this link https://cwiki.apache.org/OOZIE/map-reduce-cookbook.html but still no luck. If any1 has come across this issue, please guide me as to where i am going wrong.

Thanks in advance.

Was it helpful?

Solution

Issue resolved... While using new mapreduce API..we need to prefix the "$" symbol to the mapper and reducer class name:

<property>
 <name>mapreduce.map.class</name>
 <value>oozie.WordCount$Map</value>
</property> 
<property> 
 <name>mapreduce.reduce.class</name>
 <value>oozie.WordCount$Reduce</value>
</property>
Licensed under: CC-BY-SA with attribution
Not affiliated with StackOverflow
scroll top