Monday 28 July 2014

Oozie Coordinator to Schedule Sqoop Data Everyday

1. coordinator.properties

nameNode=hdfs://localhost.localdomain:8020
jobTracker=localhost.localdomain:8021
queueName=default


oozie.libpath=${nameNode}/user/oozie/share/lib
oozie.use.system.libpath=true
oozie.wf.rerun.failnodes=true


startDay=2014-07-28T16:05Z
endDay=2014-07-30T23:59Z


appPath=${nameNode}/user/${user.name}/simple_workflow

oozie.coord.application.path=${appPath}

optionFile=sqoop_param.txt

DataPath=${nameNode}/user/${user.name}/RawData

2. Coordinator.xml
<coordinator-app name="silverdollar-scheduler" xmlns="uri:oozie:coordinator:0.2"
                 frequency="${coord:days(1)}" start="${startDay}" end= "${endDay}"
                 timezone="America/New_York">

    <controls>
        <timeout>5</timeout>
    </controls>

    <action>
        <workflow>
            <app-path>${appPath}</app-path>
        </workflow>
    </action>
</coordinator-app>

3. workflow.xml
<workflow-app name="Oozie" xmlns="uri:oozie:workflow:0.4">
    <start to="ImportSqoop"/>
 
    <action name="ImportSqoop">
        <sqoop xmlns="uri:oozie:sqoop-action:0.2">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path='${DataPath}'/>
            </prepare>
            <configuration>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
            <arg>--options-file</arg>
            <arg>${optionFile}</arg>
            <file>${optionFile}</file>
        </sqoop>
        <ok to="End"/>
        <error to="Kill"/>
    </action>

    <kill name="Kill">
        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>

    <end name="End"/>
</workflow-app>

4. command
oozie job -oozie http://localhost:11000/oozie -config  simple_workflow/coordinator.properties  -run

Reference:
1. http://www.thecloudavenue.com/2013/11/CreatingSimpleCoordinatorUsingOozie.html
2. http://oozie.apache.org/docs/3.3.2/CoordinatorFunctionalSpec.html

No comments:

Post a Comment