The "Secondary Logon" service is not running. This service is required for the installer to initialize the database. Please start this service and try again.
home-> Control panel -> administrative tools -> services
> use week6 > db.trades.findOne() { "_id" : ObjectId("51b05fbce3600f7b48448eda"), "ticker" : "abcd", "time" : ISODate("2012-03-03T04:24:13.003Z"), "price" : 110, "shares" : 200, "ticket" : "z135" // this key used for group in pig script }
root@boss[bin]#vi cntWthr.pig
.....-- MongoDB Java driver
REGISTER /opt/hadoop-1.1.0/lib/mongo-2.10.1.jar;
-- Core Mongo-Hadoop Library
REGISTER /opt/bigdata/mongo-hadoop/core/target/mongo-hadoop-core_1.1.2-1.1.0.jar;
-- mongo-hadoop pig support
REGISTER /opt/bigdata/mongo-hadoop/pig/target/mongo-hadoop-pig_1.1.2-1.1.0.jar;
trades = LOAD 'mongodb://localhost:27017/week6.trades' using com.mongodb.hadoop.pig.MongoLoader;
grp = GROUP trades by $0#'ticket';
cnt = FOREACH grp GENERATE group,COUNT(trades);
--dump cnt;
STORE cnt INTO 'mongodb://localhost:27017/mongo_hadoop.yield_historical.outt' USING com.mongodb.hadoop.pig.MongoInsertStorage('group:float,cnt:int', 'group');
root@boss[bin]#vi mongoHadoopNode.js
// This script must be run from PIG_HOME/bin and file cntWthr.pig must be exist in same path.
var express = require('express'),
app = express(),
cons = require('consolidate'),
mongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server;// Configuring view template
app.engine('html',cons.swig);
app.set('view engine','html');
app.set('views', __dirname + "/views");
// Running pig script file
var spawn = require('child_process').spawn,
runPig = spawn('pig',['cntWthr.pig']);
// Handling the output
runPig.stdout.on('data',function(data){
console.log('stdout : '+data);
});
runPig.stderr.on('data',function(data){
console.log('stderr : '+data );
});
app.get('/',function(req,res){
mongoClient.connect('mongodb://localhost:27017/mongo_hadoop',function(err,db) {
if(err) throw err;
db.collection('yield_historical.outt').findOne({},function(err,doc){
res.render('template',{'Group':doc.group,'Value':doc.val_0});
db.close();
});;
});
});
app.get('*',function(req,res){
res.send("Page Not Found !!!! ");
});
mongo.open(function(err,mongo){
if(err) throw err;
app.listen(8000);
console.log("Express server started successfully localhost:8000")
});
MongoDB-Hadoop integraton and Job processing with PIG
Group : {{Group}} Value : {{Value}}
root@boss[bin]#node mongoHadoopNode.js Express server started successfully localhost:8000
> use mongo_hadoop> db.yield_historical.outt.findOne(){ "_id" : ObjectId("521f0ebf908dfe1853af7c01"), "group" : "z135", "val_0" : NumberLong(1667) }
http://localhost:8000
MongoDB-Hadoop integraton and Job processing with PIG
Group : z447
Value : 834
root@boss:/opt/bigdata/pig-0.11.1/bin>vi runPig.js
var spawn = require('child_process').spawn;var runPig = spawn('pig',['cntWthr.pig']);runPig.stdout.on('data',function(data){console.log('stdout : '+data);});runPig.stderr.on('data',function(data){console.log('stderr : '+data + " process Home : "+process.env.HOME);});
root@boss:/opt/bigdata/pig-0.11.1/bin>node runPig.js
root@boss:/opt/bigdata/pig-0.11.1/bin>vi runPig.js
var sys = require('sys');var exec = require('child_process').exec;function puts(error, stdout, stderr) { sys.puts(stdout) }exec("pig -f cntWthr.pig", puts);
root@boss:/opt/bigdata/pig-0.11.1/bin>node runPig.js
root@boss[mongo-hadoop]# ./sbt package
java.lang.RuntimeException: Hadoop Release '%s' is an invalid/unsupported release. Valid entries are in 0.21.0 at scala.sys.package$.error(package.scala:27) at MongoHadoopBuild$$anonfun$streamingSettings$6$$anonfun$apply$8.apply(MongoHadoopBuild.scala:176) at MongoHadoopBuild$$anonfun$streamingSettings$6$$anonfun$apply$8.apply(MongoHadoopBuild.scala:176) at scala.collection.MapLike$class.getOrElse(MapLike.scala:122) at scala.collection.immutable.HashMap.getOrElse(HashMap.scala:38) at MongoHadoopBuild$$anonfun$streamingSettings$6.apply(MongoHadoopBuild.scala:176) at MongoHadoopBuild$$anonfun$streamingSettings$6.apply(MongoHadoopBuild.scala:175) at scala.Function1$$anonfun$compose$1.apply(Function1.scala:49) at scala.Function1$$anonfun$compose$1.apply(Function1.scala:49) at sbt.EvaluateSettings$$anonfun$sbt$EvaluateSettings$$single$1.apply(INode.scala:159) at sbt.EvaluateSettings$$anonfun$sbt$EvaluateSettings$$single$1.apply(INode.scala:159) at sbt.EvaluateSettings$MixedNode.evaluate0(INode.scala:177) at sbt.EvaluateSettings$INode.evaluate(INode.scala:132) at sbt.EvaluateSettings$$anonfun$sbt$EvaluateSettings$$submitEvaluate$1.apply$mcV$sp(INode.scala:64) at sbt.EvaluateSettings.sbt$EvaluateSettings$$run0(INode.scala:73) at sbt.EvaluateSettings$$anon$3.run(INode.scala:69) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) [error] Hadoop Release '%s' is an invalid/unsupported release. Valid entries are in 0.21.0 [error] Use 'last' for the full log.
"0.21.0" to "0.21"
root@boss[mongo-hadoop]#vi build.sbt ....... ....... hadoopRelease in ThisBuild := "0.21"
root@boss[mongo-hadoop]# ./sbt package
module not found: org.apache.hadoop#hadoop-core;0.21.0 [warn] ==== local: tried [warn] /root/.ivy2/local/org.apache.hadoop/hadoop-core/0.21.0/ivys/ivy.xml [warn] ==== Simile Repo at MIT: tried [warn] http://simile.mit.edu/maven/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [warn] ==== Cloudera Repository: tried [warn] https://repository.cloudera.com/artifactory/cloudera-repos/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [warn] ==== Maven.Org Repository: tried [warn] http://repo1.maven.org/maven2/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [warn] ==== releases: tried [warn] https://oss.sonatype.org/content/repositories/releases/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [warn] ==== public: tried [warn] http://repo1.maven.org/maven2/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [info] Resolving org.specs2#specs2-scalaz-core_2.9.2;6.0.1 ... [warn] :::::::::::::::::::::::::::::::::::::::::::::: [warn] :: UNRESOLVED DEPENDENCIES :: [warn] :::::::::::::::::::::::::::::::::::::::::::::: [warn] :: org.apache.hadoop#hadoop-core;0.21.0: not found [warn] :::::::::::::::::::::::::::::::::::::::::::::: sbt.ResolveException: unresolved dependency: org.apache.hadoop#hadoop-core;0.21.0: not found
grunt> REGISTER /opt/bigdata/mongo-hadoop/piggybank-0.3-amzn.jar ...... grunt> date_tenyear = foreach raw generate UnixToISO($0#'_id'), $0#'bc10Year';
ERROR 1070: Could not resolve org.apache.pig.piggybank.evaluation.datetime.convert.UnixToISO using imports: [, org.apache.pig.builtin., org.apache.pig.impl.builtin.]
grunt> REGISTER /opt/bigdata/pig-0.10.0/contrib/piggybank/java/piggybank.jar; ....
