<?xml version="1.0" encoding="UTF-8"?>
<!--
  Licensed to the Apache Software Foundation (ASF) under one or more
  contributor license agreements.  See the NOTICE file distributed with
  this work for additional information regarding copyright ownership.
  The ASF licenses this file to You under the Apache License, Version 2.0
  (the "License"); you may not use this file except in compliance with
  the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
-->
<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">

<document>
  <header>
    <title>POST mapreduce/streaming </title>
  </header>

  <body>

  <section>
   <title>Description</title>
    <p>Create and queue an
     <a href="http://hadoop.apache.org/common/docs/current/streaming.html">Hadoop
      streaming MapReduce</a> job.</p>
  </section>

  <section>
   <title>URL</title>
    <p><code>http://</code>www.myserver.com<code>/templeton/v1/mapreduce/streaming</code></p>
  </section>

  <section>
   <title>Parameters</title>
    <table>
    <tr><th>Name</th><th>Description</th><th>Required?</th><th>Default</th></tr>

    <tr><td><strong>input</strong></td>
        <td>Location of the input data in Hadoop.</td>
        <td>Required</td>
        <td>None</td>
    </tr>

    <tr><td><strong>output</strong></td>
        <td>Location in which to store the output data.  If not specified,
            Templeton will store the output in a location that can be discovered
            using the <a href="queue.html">queue</a> resource.</td>
        <td>Optional</td>
        <td>See description</td>
    </tr>

    <tr><td><strong>mapper</strong></td>
        <td>Location of the mapper program in Hadoop.</td>
        <td>Required</td>
        <td>None</td>
    </tr>

    <tr><td><strong>reducer</strong></td>
        <td>Location of the reducer program in Hadoop.</td>
        <td>Required</td>
        <td>None</td>
    </tr>

    <tr><td><strong>file</strong></td>
        <td>Add an HDFS file to the distributed cache.</td>
        <td>Optional</td>
        <td>None</td>
    </tr>

    <tr><td><strong>define</strong></td>
        <td>Set an Hadoop configuration variable using the syntax
         <code>define=NAME=VALUE</code></td>
        <td>Optional</td>
        <td>None</td>
    </tr>

    <tr><td><strong>cmdenv</strong></td>
        <td>Set an environment variable using the syntax
         <code>cmdenv=NAME=VALUE</code></td>
        <td>Optional</td>
        <td>None</td>
    </tr>

    <tr><td><strong>arg</strong></td>
        <td>Set a program argument.</td>
        <td>Optional</td>
        <td>None</td>
    </tr>

    <tr><td><strong>statusdir</strong></td>
        <td>A directory where Templeton will write the status of the
         Map Reduce job.  If provided, it is the caller's responsibility
         to remove this directory when done.</td>
        <td>Optional</td>
        <td>None</td>
    </tr>

    <tr><td><strong>enablelog</strong></td>
        <td>Collecting hadoop job config and logs into $statusdir/logs folder.
         statusdir must be set as well to use this feature.</td>
        <td>Optional</td>
        <td>None</td>
    </tr>

    <tr><td><strong>callback</strong></td>
        <td>Define a URL to be called upon job completion. You may embed a specific
         job ID into this URL using <code>$jobId</code>.  This tag
         will be replaced in the callback URL with this job's job ID.</td>
        <td>Optional</td>
        <td>None</td>
    </tr>

    </table>
  </section>

  <section>
   <title>Results</title>
     <table>
      <tr><th>Name</th><th>Description</th></tr>

      <tr><td><strong>id</strong></td>
        <td>A string containing the job ID similar to "job_201110132141_0001".</td>
      </tr>

      <tr><td><strong>info</strong></td>
        <td>A JSON object containing the information returned when the job was queued.
            See the Hadoop documentation
            (<a href="http://hadoop.apache.org/common/docs/stable/api/org/apache/hadoop/mapred/TaskController.html">Class
            TaskController</a>) for more information.</td>
      </tr>
     </table>
  </section>

  <section>
   <title>Example</title>

   <p><strong>Code and Data Setup</strong></p>
<source>
% cat mydata/file01 mydata/file02
Hello World Bye World
Hello Hadoop Goodbye Hadoop

% hadoop fs -put mydata/ .

% hadoop fs -ls mydata
Found 2 items
-rw-r--r--   1 ctdean supergroup         23 2011-11-11 13:29 /user/ctdean/mydata/file01
-rw-r--r--   1 ctdean supergroup         28 2011-11-11 13:29 /user/ctdean/mydata/file02
</source>

   <p><strong>Curl Command</strong></p>
<source>
% curl -s -d user.name=ctdean \
       -d input=mydata \
       -d output=mycounts \
       -d mapper=/bin/cat \
       -d reducer="/usr/bin/wc -w" \
       'http://localhost:50111/templeton/v1/mapreduce/streaming'
</source>

   <p><strong>JSON Output</strong></p>
<source>
{
 "id": "job_201111111311_0008",
 "info": {
          "stdout": "packageJobJar: [] [/Users/ctdean/var/hadoop/hadoop-0.20.205.0/share/hadoop/contrib/streaming/hadoop-streaming-0.20.205.0.jar...
                    templeton-job-id:job_201111111311_0008
                    ",
          "stderr": "11/11/11 13:26:43 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments
                    11/11/11 13:26:43 INFO mapred.FileInputFormat: Total input paths to process : 2
                    ",
          "exitcode": 0
         }
}
</source>

   <p><strong>Results</strong></p>
<source>
% hadoop fs -ls mycounts
Found 3 items
-rw-r--r--   1 ctdean supergroup          0 2011-11-11 13:27 /user/ctdean/mycounts/_SUCCESS
drwxr-xr-x   - ctdean supergroup          0 2011-11-11 13:26 /user/ctdean/mycounts/_logs
-rw-r--r--   1 ctdean supergroup         10 2011-11-11 13:27 /user/ctdean/mycounts/part-00000

% hadoop fs -cat mycounts/part-00000
      8
</source>
  </section>
  </body>
</document>
