> use week6 > db.trades.findOne() { "_id" : ObjectId("51b05fbce3600f7b48448eda"), "ticker" : "abcd", "time" : ISODate("2012-03-03T04:24:13.003Z"), "price" : 110, "shares" : 200, "ticket" : "z135" // this key used for group in pig script }
root@boss[bin]#vi cntWthr.pig
.....-- MongoDB Java driver
REGISTER /opt/hadoop-1.1.0/lib/mongo-2.10.1.jar;
-- Core Mongo-Hadoop Library
REGISTER /opt/bigdata/mongo-hadoop/core/target/mongo-hadoop-core_1.1.2-1.1.0.jar;
-- mongo-hadoop pig support
REGISTER /opt/bigdata/mongo-hadoop/pig/target/mongo-hadoop-pig_1.1.2-1.1.0.jar;
trades = LOAD 'mongodb://localhost:27017/week6.trades' using com.mongodb.hadoop.pig.MongoLoader;
grp = GROUP trades by $0#'ticket';
cnt = FOREACH grp GENERATE group,COUNT(trades);
--dump cnt;
STORE cnt INTO 'mongodb://localhost:27017/mongo_hadoop.yield_historical.outt' USING com.mongodb.hadoop.pig.MongoInsertStorage('group:float,cnt:int', 'group');
root@boss[bin]#vi mongoHadoopNode.js
// This script must be run from PIG_HOME/bin and file cntWthr.pig must be exist in same path.
var express = require('express'),
app = express(),
cons = require('consolidate'),
mongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server;// Configuring view template
app.engine('html',cons.swig);
app.set('view engine','html');
app.set('views', __dirname + "/views");
// Running pig script file
var spawn = require('child_process').spawn,
runPig = spawn('pig',['cntWthr.pig']);
// Handling the output
runPig.stdout.on('data',function(data){
console.log('stdout : '+data);
});
runPig.stderr.on('data',function(data){
console.log('stderr : '+data );
});
app.get('/',function(req,res){
mongoClient.connect('mongodb://localhost:27017/mongo_hadoop',function(err,db) {
if(err) throw err;
db.collection('yield_historical.outt').findOne({},function(err,doc){
res.render('template',{'Group':doc.group,'Value':doc.val_0});
db.close();
});;
});
});
app.get('*',function(req,res){
res.send("Page Not Found !!!! ");
});
mongo.open(function(err,mongo){
if(err) throw err;
app.listen(8000);
console.log("Express server started successfully localhost:8000")
});
MongoDB-Hadoop integraton and Job processing with PIG
Group : {{Group}} Value : {{Value}}
root@boss[bin]#node mongoHadoopNode.js Express server started successfully localhost:8000
> use mongo_hadoop> db.yield_historical.outt.findOne(){ "_id" : ObjectId("521f0ebf908dfe1853af7c01"), "group" : "z135", "val_0" : NumberLong(1667) }
http://localhost:8000
MongoDB-Hadoop integraton and Job processing with PIG
Group : z447
Value : 834
root@boss[mongo-hadoop]# ./sbt package
java.lang.RuntimeException: Hadoop Release '%s' is an invalid/unsupported release. Valid entries are in 0.21.0 at scala.sys.package$.error(package.scala:27) at MongoHadoopBuild$$anonfun$streamingSettings$6$$anonfun$apply$8.apply(MongoHadoopBuild.scala:176) at MongoHadoopBuild$$anonfun$streamingSettings$6$$anonfun$apply$8.apply(MongoHadoopBuild.scala:176) at scala.collection.MapLike$class.getOrElse(MapLike.scala:122) at scala.collection.immutable.HashMap.getOrElse(HashMap.scala:38) at MongoHadoopBuild$$anonfun$streamingSettings$6.apply(MongoHadoopBuild.scala:176) at MongoHadoopBuild$$anonfun$streamingSettings$6.apply(MongoHadoopBuild.scala:175) at scala.Function1$$anonfun$compose$1.apply(Function1.scala:49) at scala.Function1$$anonfun$compose$1.apply(Function1.scala:49) at sbt.EvaluateSettings$$anonfun$sbt$EvaluateSettings$$single$1.apply(INode.scala:159) at sbt.EvaluateSettings$$anonfun$sbt$EvaluateSettings$$single$1.apply(INode.scala:159) at sbt.EvaluateSettings$MixedNode.evaluate0(INode.scala:177) at sbt.EvaluateSettings$INode.evaluate(INode.scala:132) at sbt.EvaluateSettings$$anonfun$sbt$EvaluateSettings$$submitEvaluate$1.apply$mcV$sp(INode.scala:64) at sbt.EvaluateSettings.sbt$EvaluateSettings$$run0(INode.scala:73) at sbt.EvaluateSettings$$anon$3.run(INode.scala:69) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) [error] Hadoop Release '%s' is an invalid/unsupported release. Valid entries are in 0.21.0 [error] Use 'last' for the full log.
"0.21.0" to "0.21"
root@boss[mongo-hadoop]#vi build.sbt ....... ....... hadoopRelease in ThisBuild := "0.21"
root@boss[mongo-hadoop]# ./sbt package
module not found: org.apache.hadoop#hadoop-core;0.21.0 [warn] ==== local: tried [warn] /root/.ivy2/local/org.apache.hadoop/hadoop-core/0.21.0/ivys/ivy.xml [warn] ==== Simile Repo at MIT: tried [warn] http://simile.mit.edu/maven/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [warn] ==== Cloudera Repository: tried [warn] https://repository.cloudera.com/artifactory/cloudera-repos/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [warn] ==== Maven.Org Repository: tried [warn] http://repo1.maven.org/maven2/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [warn] ==== releases: tried [warn] https://oss.sonatype.org/content/repositories/releases/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [warn] ==== public: tried [warn] http://repo1.maven.org/maven2/org/apache/hadoop/hadoop-core/0.21.0/hadoop-core-0.21.0.pom [info] Resolving org.specs2#specs2-scalaz-core_2.9.2;6.0.1 ... [warn] :::::::::::::::::::::::::::::::::::::::::::::: [warn] :: UNRESOLVED DEPENDENCIES :: [warn] :::::::::::::::::::::::::::::::::::::::::::::: [warn] :: org.apache.hadoop#hadoop-core;0.21.0: not found [warn] :::::::::::::::::::::::::::::::::::::::::::::: sbt.ResolveException: unresolved dependency: org.apache.hadoop#hadoop-core;0.21.0: not found
grunt> REGISTER /opt/bigdata/mongo-hadoop/piggybank-0.3-amzn.jar ...... grunt> date_tenyear = foreach raw generate UnixToISO($0#'_id'), $0#'bc10Year';
ERROR 1070: Could not resolve org.apache.pig.piggybank.evaluation.datetime.convert.UnixToISO using imports: [, org.apache.pig.builtin., org.apache.pig.impl.builtin.]
grunt> REGISTER /opt/bigdata/pig-0.10.0/contrib/piggybank/java/piggybank.jar; ....
root@boss[bin]#/etc/init.d/mongodb stop
Stopping database: mongodb.
root@boss[bin]#vi /etc/mongodb.conf ..... ..... #dbpath=/var/lib/mongodb dbpath=/data/db ..... .....
root@boss[bin]#/etc/init.d/mongodb strat
root@boss[bin]#tail -f /var/log/mongodb/mongodb.log
***** SERVER RESTARTED *****
Tue Jun 18 15:32:41.170 [initandlisten] MongoDB starting : pid=20270 port=27017 dbpath=/data/db 32-bit host=boss
Tue Jun 18 15:32:41.170 [initandlisten]
Tue Jun 18 15:32:41.170 [initandlisten] ** NOTE: This is a 32 bit MongoDB binary.
Tue Jun 18 15:32:41.170 [initandlisten] ** 32 bit builds are limited to less than 2GB of data (or less with --journal).
Tue Jun 18 15:32:41.170 [initandlisten] ** Note that journaling defaults to off for 32 bit and is currently off.
Tue Jun 18 15:32:41.170 [initandlisten] ** See http://dochub.mongodb.org/core/32bit
Tue Jun 18 15:32:41.170 [initandlisten]
Tue Jun 18 15:32:41.170 [initandlisten] db version v2.4.4
Tue Jun 18 15:32:41.170 [initandlisten] git version: 4ec1fb96702c9d4c57b1e06dd34eb73a16e407d2
Tue Jun 18 15:32:41.170 [initandlisten] build info: Linux bs-linux32.10gen.cc 2.6.21.7-2.fc8xen #1 SMP Fri Feb 15 12:39:36 EST 2008 i686 BOOST_LIB_VERSION=1_49
Tue Jun 18 15:32:41.170 [initandlisten] allocator: system
Tue Jun 18 15:32:41.170 [initandlisten] options: { config: "/etc/mongodb.conf", dbpath: "/data/db", logappend: "true", logpath: "/var/log/mongodb/mongodb.log" }
Tue Jun 18 15:32:41.170 [initandlisten] exception in initAndListen: 10309 Unable to create/open lock file: /data/db/mongod.lock errno:13 Permission denied Is a mongod instance already running?, terminating
Tue Jun 18 15:32:41.170 dbexit:
Tue Jun 18 15:32:41.170 [initandlisten] shutdown: going to close listening sockets...
Tue Jun 18 15:32:41.170 [initandlisten] shutdown: going to flush diaglog...
Tue Jun 18 15:32:41.170 [initandlisten] shutdown: going to close sockets...
Tue Jun 18 15:32:41.170 [initandlisten] shutdown: waiting for fs preallocator...
Tue Jun 18 15:32:41.170 [initandlisten] shutdown: closing all files...
Tue Jun 18 15:32:41.171 [initandlisten] closeAllFiles() finished
Tue Jun 18 15:32:41.171 [initandlisten] shutdown: removing fs lock...
Tue Jun 18 15:32:41.171 [initandlisten] couldn't remove fs lock errno:9 Bad file descriptor
Tue Jun 18 15:32:41.171 dbexit: really exiting now
its failed to start, have checked log file in (/var/log/mongodb/mongodb.log)got above error.root@boss[bin]#ls -l /var/lib/mongodb/
total 32808
-rw------- 1 mongodb nogroup 16777216 Jun 18 15:33 local.0
-rw------- 1 mongodb nogroup 16777216 Jun 18 15:33 local.ns
-rwxr-xr-x 1 mongodb mongodb 0 Jun 18 15:40 mongod.lock
root@boss[bin]#chown -R mongodb:nogroup /data/db root@boss[bin]#chown mongodb:mongodb /data/db/mongod.lockthen I restart the mongodb by /etc/init.d/mongo start ( step 3 ). this time server started successfully.
root@boss[bin]#/etc/init.d/mongodb strat
Starting database: mongodb.
root@boss[bin]#mongo MongoDB shell version: 2.4.4 connecting to: test > db test >show dbs admin 0.0625GB local 0.03125GB rs 0.0625GB snps1 0.0625GB test 0.0625GB week4 0.0625GB week5 0.0625GB week6 0.999755859375GB
root@boss[bin]#apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10
10gen package authenticated and it so require GPG key.root@boss[bin]#echo 'deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen' | sudo tee /etc/apt/sources.list.d/10gen.list
above command will create the file which contain package location latest version of mongoDB.root@boss[bin]#sudo apt-get update
re-synchronize the package index files from their sources........ ....... : GPG error: http://downloads-distro.mongodb.org dist Release: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 9ECBEC467F0CEB10
root@boss[bin]#sudo apt-get install mongodb-10gen
latest version of mongoDB automatically installed.....
Reading package lists... Done
Building dependency tree
Reading state information... Done
You might want to run `apt-get -f install' to correct these:
The following packages have unmet dependencies:
libboost-dev: Depends: libboost1.49-dev but it is not installable
mongodb: Depends: mongodb-dev but it is not going to be installed
Depends: mongodb-server (>= 1:2.4.1-2) but it is not going to be installed
mongodb-10gen: Conflicts: mongodb but 1:2.4.3-1 is to be installed
E: Unmet dependencies. Try 'apt-get -f install' with no packages (or specify a solution).
4.1) Install any dependencies that your previous installs missed.mongodb-dev & mongodb-server. issue the blow command to install those depends. root@boss[bin]#apt-get -f install
....... ....... Install these packages without verification [y/N]? y Get:1 http://downloads-distro.mongodb.org dist/10gen mongodb-10gen 2.4.4 [86.8MB] Fetched 86.8MB in 18s (4,718kB/s) Selecting previously deselected package mongodb-10gen. (Reading database ... 154044 files and directories currently installed.) Unpacking mongodb-10gen (from .../mongodb-10gen_2.4.4_i386.deb) ... Processing triggers for man-db ... Setting up mongodb-10gen (2.4.4) ... insserv: warning: script 'K01postgresql-9.0' missing LSB tags and overrides insserv: warning: script 'K01EnterpriseDBApachePhp' missing LSB tags and overrides insserv: warning: script 'postgresql-9.0' missing LSB tags and overrides insserv: warning: script 'EnterpriseDBApachePhp' missing LSB tags and overrides Starting database: mongodb.by issuing above command mongoDB server installed & started successfully.
root@boss[bin]#ps -ef | grep mongo
mongodb 14603 1 0 10:53 ? 00:00:00 /usr/bin/mongod --config /etc/mongodb.conf
root 14655 18627 0 10:54 pts/3 00:00:00 grep mongo
also use root@boss[bin]#ps -A | grep mongo
14603 ? 00:00:00 mongod
5.2) Get the binary executable location of mongoDBroot@boss[bin]#whereis mongo
mongo: /usr/bin/mongo /usr/share/man/man1/mongo.1.gz
root@boss[bin]#/etc/init.d/mongodb strat
Starting database: mongodb.
get the status root@boss[bin]#/etc/init.d/mongodb status
Checking status of database: mongodb running..
stopping mongoDB root@boss[bin]#/etc/init.d/mongodb stop
Stopping database: mongodb.
7) Running mongoroot@boss[bin]#mongo
MongoDB shell version: 2.4.4
connecting to: test
> db
test
>
by default its connecting test database.select* from pg_stat_activity
postgres=# select datname, usename, application_name, client_addr,client_port,query_start,current_query from pg_stat_activity
+------+---------+------------------------------+-------+---------+------+-------+------------------+
|datname |usename | application_name | client_addr | client_port | query_start
+------+---------+------------------------------+-------+---------+------+-------+------------------+
template1 | postgres | pgAdmin III - Browser | 127.0.0.1 | 56019 | 2013-06-14 19:52:52.775344+05:30
SAAS | postgres | psql | 192.168.1.10 | 58776 | 2013-06-14 19:58:33.084661+05:30
koha | postgres | pgAdmin III - Browser | 192.168.0.2 | 56020 | 2013-06-14 19:52:52.729838+05:30
pis | postgres | pgAdmin III - Query Tool | 127.0.0.1 | 56022 | 2013-06-14 19:56:21.729744+05:30-
- It display the list of active user/client ip address and query details.show processlist; mysql> show processlist; +------+---------+------------------------------+-------+---------+------+-------+------------------+ | Id | User | Host | db | Command | Time | State | Info | +------+---------+------------------------------+-------+---------+------+-------+------------------+ | 3832 | root | localhost | mysql | Query | 0 | NULL | show processlist | | 3834 | erbnext | localhost | NULL | Sleep | 2524 | | NULL | | 3837 | root | solaimurugan.chennai.in:36125| koha | Sleep | 3 | | NULL | +------+---------+------------------------------+-------+---------+------+-------+------------------+ 3 rows in set (0.00 sec)
localhost & 1 from client solaimurugan.chennai.in & it connects database mysql & koha.------------
db.currentOp(true)----------------------------------------------------------------------------------------------------------------------root@boss[bin]#./mongo MongoDB shell version: 2.2.3 ----------------------------------------------------------------------------------------------------------------------- db.currentOp(true).inprog.forEach(function(d){if(d.client && d.client!="0.0.0.0:0")printjson(d.client)})"solaimurugan.chennai.in:56231""127.0.0.1:49563""192.168.31.101:50132"-------------------------------------------------------------------------------------------------------------------------------------
mongoDB also has command to view total number of current connection & available connection to be established by client
db.serverStatus()-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------root@boss[bin]#./mongo MongoDB shell version: 2.2.3 ----------------------------------------------------------------------------------------------------------------------- db.serverStatus().connections{ "current" : 3, "available" : 816 }