Quantcast
Channel: Hortonworks » All Replies
Viewing all 3435 articles
Browse latest View live

HMaster and HRegionServer not working

$
0
0

Hello All, I had installed Hadoop 2.6.0 with Hbase 1.0. Have configured all the settings mentioned in the Apache Website for both hadoop and Hbase. Hadoop runs successfully, but when I Start HBase, I dont see both HMaster and HRegionServer in JPS. Please find below my config files:

<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://hadoopmaster:9000/hbase</value>
</property>
<property>
<name>hbase.tmp.dir</name>
<value>/tmp</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>localhost</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
</property>
<property>
<name>hbase.zookeeper.property.dataDir</name>
<value>/home/yarn/hbase/zookeeper</value>
</property>
<property>
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
<property>
<name>hbase.master</name>
<value>hadoopmaster:60000</value>
</property>
</configuration>

*********************************************

/etc/hosts

127.0.0.1 localhost hadoopmaster

# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

Have been workin on this issue for past 2 days. Please let me know what is wrong?

Regards,


Sandbox : Hive Hbase Integration Problem

$
0
0

Hi,
I am using a sandbox Virtual Machine and trying to migrate data into HIVE_ON_HBASE, ie., creating table on hive and inserting values into hbase using hbase api, therefore retrieving data from hive using following code :

import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;

import org.apache.commons.configuration.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;

public class HiveHbaseIntegration {
private static String driverName = “org.apache.hive.jdbc.HiveDriver”;

public static void main(String[] args) throws SQLException, IOException {
Connection con = null;
try {
Class.forName(driverName);
con = DriverManager.getConnection(“jdbc:hive2://192.168.135.133:10000/default”,”hive”,”hive”);
System.out.println(“connection is “+con);
Statement stmt = con.createStatement();
String tableName = “sample123″;
stmt.execute(“drop table if exists ” + tableName);
stmt.close();
String a=”create table ” + tableName + ” (eid string,ename string,esal string,deptid string) ” +
“STORED BY \’org.apache.hadoop.hive.hbase.HBaseStorageHandler\’ ” +
“WITH SERDEPROPERTIES(\’hbase.columns.mapping\’ = \':key,sample123:ename,sample123:esal,sample123:deptid\’) ” +
“TBLPROPERTIES(\’hbase.table.name\’ = \’sample123\’)”;

stmt = con.createStatement();
stmt.execute(a);
stmt.close();
System.out.println(“table created “);
org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
conf.set(“hbase.defaults.for.version.skip”, “true”);
conf.set(“hbase.zookeeper.quorum”, “192.168.135.133”);

conf.addResource(conf);
HBaseAdmin.checkHBaseAvailable(conf);
System.out.println(“HBase is Available and is Running “);

System.out.println(“confuguration for hbase is”+conf);
HTable table = new HTable(conf, “sample123″);
Put p=new Put(Bytes.toBytes(“1″));
System.out.println(“row in table”+p);
p.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“ename”),Bytes.toBytes(“solix”));
p.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“esal”),Bytes.toBytes(“1000″));
p.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“deptid”),Bytes.toBytes(“1000″));
System.out.println(“After populating Put “+p);

System.out.println(“row in table”+p);
Put p1=new Put(Bytes.toBytes(“2″));
p1.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“ename”),Bytes.toBytes(“solix1″));
p1.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“esal”),Bytes.toBytes(“10001″));
p1.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“deptid”),Bytes.toBytes(“10001″));
Put p3=new Put(Bytes.toBytes(“3″));
p3.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“ename”),Bytes.toBytes(“solix12″));
p3.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“esal”),Bytes.toBytes(“100011″));
p3.add(Bytes.toBytes(“sample123″), Bytes.toBytes(“deptid”),Bytes.toBytes(“100011″));
table.put(p);
table.put(p1);
table.put(p3);
System.out.println(“After table Put “+p);
table.flushCommits();
System.out.println(“FLUSH COMMITS EXECUTED”);

} catch (Exception e) {
e.printStackTrace();
}finally {
// con.close();
}

}
}

Where i am getting the exception is

org.apache.hadoop.hbase.MasterNotRunningException: org.apache.hadoop.hbase.MasterNotRunningException: The node /hbase is not in ZooKeeper. It should have been written by the master. Check the value configured in ‘zookeeper.znode.parent’. There could be a mismatch with the one configured in the master.
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1661)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(ConnectionManager.java:1687)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getKeepAliveMasterService(ConnectionManager.java:1904)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.isMasterRunning(ConnectionManager.java:932)
at org.apache.hadoop.hbase.client.HBaseAdmin.checkHBaseAvailable(HBaseAdmin.java:2409)
at connections.HiveHbaseIntegration.main(HiveHbaseIntegration.java:47)
Caused by: org.apache.hadoop.hbase.MasterNotRunningException: The node /hbase is not in ZooKeeper. It should have been written by the master. Check the value configured in ‘zookeeper.znode.parent’. There could be a mismatch with the one configured in the master.
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(ConnectionManager.java:912)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.access$600(ConnectionManager.java:587)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(ConnectionManager.java:1586)
at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation$StubMaker.makeStub(ConnectionManager.java:1633)
… 5 more

please help me to overcome the issue.

Reply To: Unable to instantiate UDF implementation class

$
0
0

So, i added the udf jar to our Dev cluster which has hadoop 2.4.0.2.1 and it works well there.

It does not still work on the Sandbox which has hadoop 2.6.0.2.2. I have made sure that I used the Hadoop and Hive jar files from the Sandbox.

Any input is appreciated. Thanks.

Error Importing sql sever data from sqoop

$
0
0

Hi

I am importing data from sql server to hdfs and below is the command

sqoop import –connect “jdbc:sqlserver://Servername:1433;username=hadoop;password=Password;database=MSBI” –table DimDate –target-dir /Hadoop/hdpdatadn/dn/DW/msbi

but I am getting following error:

User: amit.tomar
Name: DimDate.jar
Application Type: MAPREDUCE
Application Tags:
State: FAILED
FinalStatus: FAILED
Started: Wed May 27 12:39:48 +0800 2015
Elapsed: 23sec
Tracking URL: History
Diagnostics: Application application_1432698911303_0005 failed 2 times due to AM Container for appattempt_1432698911303_0005_000002 exited with exitCode: 1
For more detailed output, check application tracking page:http://apsgsvdevsql01.ap.didata.local:8088/proxy/application_1432698911303_0005/Then, click on links to logs of each attempt.
Diagnostics: Exception from container-launch.
Container id: container_1432698911303_0005_02_000001
Exit code: 1
Stack trace: ExitCodeException exitCode=1:
at org.apache.hadoop.util.Shell.runCommand(Shell.java:538)
at org.apache.hadoop.util.Shell.run(Shell.java:455)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:715)
at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:212)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:302)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:82)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Shell output: 1 file(s) moved.
Container exited with a non-zero exit code 1
Failing this attempt. Failing the application.

From the log below is the message:

java.lang.Exception: Unknown container. Container either has not started or has already completed or doesn’t belong to this node at all.

Thanks in advance

Reply To: /hdp/apps folder is not getting created while installing HDP2.2 using Ambari 1.7

Reply To: Spark 1.3 Upgarde on HDP-2.2.4.2-2

$
0
0

Thanks I will try that.

Please update the document to add this line also

hdp-select set spark-client 2.2.4.4-16

optimal size for hbase.hregion.memstore.flush.size

$
0
0

Hi all,

We have a very high write heavy application. I would like to set a optimal value for the hbase.hregion.memstore.flush.size. It is mentioned that the size should be between 128 and 256 Mb.

I would like to know if I could set this to higher value, like 512 Mb or even 1 GB ? What is the impact of having this value higher?

Reply To: Phoenix/hbase client

$
0
0

it dont find class driver jdbc phoenix

>>>Caused by: java.lang.ClassNotFoundException: org.apache.phoenix.jdbc.PhoenixDriver

you need to add the phoenix client jar contain the driver phoenix, for your client, it must be visible in classpath

>>>add the phoenix client jar to the classpath of your HBase client


Reply To: HMaster and HRegionServer not working

$
0
0

Any help would be greatly appreciated.

Unable to create Hbase table using Hive query through Spark

$
0
0

Using the following tutorial, I was able to do the HBase integration with Hive. After the configuration I was successfully able to create Hbase table using Hive query with Hive table mapping.

Hive query:


CREATE TABLE upc_hbt(key string, value string) 
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,value:value")
TBLPROPERTIES ("hbase.table.name" = "upc_hbt");

Spark-Scala:


val createTableHql : String = s"CREATE TABLE upc_hbt2(key string, value string)"+
      "STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'"+
      "WITH SERDEPROPERTIES ('hbase.columns.mapping' = ':key,value:value')"+
      "TBLPROPERTIES ('hbase.table.name' = 'upc_hbt2')"

    hc.sql(createTableHql)

But when I execute the same Hive query through Spark it throws the following error:


Exception in thread "main" org.apache.spark.sql.execution.QueryExecutionException: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: Error in loading storage handler.org.apache.hadoop.hive.hbase.HBaseStorageHandler

It’s seem like during the Hive execution through Spark it can’t find the auxpath jar location. Is there anyway to solve this problem?

Thank you very much in advance.

Sqoop for AS400

$
0
0

Hi I am trying to connect to an IBM iseries AS400 server

Command:
sqoop list-tables –driver com.ibm.as400.access.AS400JDBCDataSource –connect jdbc:as400://server:992/–username=name –password=pass

Warning: /usr/lib/sqoop/../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
15/05/28 04:22:13 INFO sqoop.Sqoop: Running Sqoop version: 1.4.4.2.1.1.0-385
15/05/28 04:22:13 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
15/05/28 04:22:13 WARN sqoop.ConnFactory: Parameter –driver is set to an explicit driver however appropriate connection manager is not being set (via –connection-manager). Sqoop is going to fall back to org.apache.sqoop.manager.GenericJdbcManager. Please specify explicitly which connection manager should be used next time.
15/05/28 04:22:13 INFO manager.SqlManager: Using default fetchSize of 1000
15/05/28 04:22:13 ERROR manager.SqlManager: Error reading database metadata: java.sql.SQLException: No suitable driver found for jdbc:as400://11.143.116.29:992/–username=rpceod
java.sql.SQLException: No suitable driver found for jdbc:as400://11.143.116.29:992/–username=rpceod
at java.sql.DriverManager.getConnection(DriverManager.java:596)
at java.sql.DriverManager.getConnection(DriverManager.java:233)
at org.apache.sqoop.manager.SqlManager.makeConnection(SqlManager.java:824)
at org.apache.sqoop.manager.GenericJdbcManager.getConnection(GenericJdbcManager.java:52)
at org.apache.sqoop.manager.SqlManager.listTables(SqlManager.java:466)
at org.apache.sqoop.tool.ListTablesTool.run(ListTablesTool.java:49)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:222)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:231)
at org.apache.sqoop.Sqoop.main(Sqoop.java:240)
Could not retrieve tables list from server
15/05/28 04:22:13 ERROR tool.ListTablesTool: manager.listTables() returned null

Get information using the commandline

$
0
0

Hi,

there is plan to upgrade the Ambari cluster. I searched the documentation for some commandline examples to get some basic information without using the web at x.x.x.x:8080/login

So i’m interested in

– Ambari Version
– Hadoop Stack version
– Which services are installed
– where can i find those services (nodes)

Reply To: sqoop as400 communication

$
0
0

Hi could you share the procedure you are using to import data.

Store data to separate datanodes and grant access to specific datanodes

$
0
0

Hi,
I have several applications using the same hadoop-cluster as data-storage. The different data of each application is stored directly at HDFS or at HDFS by Hive. I want to separate the data of each application “physical” at the cluster like {DataApplication1->DataNode1…5, DataApplication2->DataNode6…10, and so on}. Additional I’m searching for a possibility to grant/denied access to data-nodes like {ServiceUserApplication1->Access to DataNode1…5, ServiceUserApplication2->Access to DataNode6…10, and so on}.
For future the usecase may change to one big data pool for all applications. Then the data will be partitioned by Hive by application (e.g. partitioned by ApplicationID) and the need will be to store partitions at specific data nodes with access to this data nodes (partitoned data) for the ServiceUsers.

To solve the first part I found this Link, but don’t know if it is as dynamic as I need it.

Are there some features to support this usecases?

HBase Issue | google protobuf tag mismatch error while deserialising SCAN st

$
0
0

Context: I am in the process of migrating my MR jobs on HBase from CDH 2.0.0-cdh4.5.0 (Hadoop1) to HDP 2.2.0.0-2041 (YARN).
After minor changes the code was compiled against HDP 2.2.0.0-2041.

Problem: I am trying to run a oozie workflow that executes a series of MR jobs after creating a scan on HBase. The scan is
created programatically and then serialised-deserialised before handing it to the mapper to fetch batches from HBase.

Issue: When TableInputFormat internally tries to deserialise the scan string, it throws an error indicating that under
the hood google protobuf was not able to deserialise the string. The stack trace looks
as follows.

Exception in thread “main” java.io.IOException: com.google.protobuf.InvalidProtocolBufferException: Protocol message end-group
tag did not match expected tag.
at com.flipkart.yarn.test.TestScanSerialiseDeserialise.convertStringToScan(TestScanSerialiseDeserialise.java:37)
at com.flipkart.yarn.test.TestScanSerialiseDeserialise.main(TestScanSerialiseDeserialise.java:25)
Caused by: com.google.protobuf.InvalidProtocolBufferException: Protocol message end-group tag did not match expected tag.
at com.google.protobuf.InvalidProtocolBufferException.invalidEndTag(InvalidProtocolBufferException.java:94)
at com.google.protobuf.CodedInputStream.checkLastTagWas(CodedInputStream.java:124)
at com.google.protobuf.CodedInputStream.readGroup(CodedInputStream.java:241)
at com.google.protobuf.UnknownFieldSet$Builder.mergeFieldFrom(UnknownFieldSet.java:488)
at com.google.protobuf.GeneratedMessage.parseUnknownField(GeneratedMessage.java:193)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$Scan.<init>(ClientProtos.java:13718)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$Scan.<init>(ClientProtos.java:13676)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$Scan$1.parsePartialFrom(ClientProtos.java:13868)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$Scan$1.parsePartialFrom(ClientProtos.java:13863)
at com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:141)
at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:176)
at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:188)
at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:193)
at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:49)
at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$Scan.parseFrom(ClientProtos.java:14555)
at com.flipkart.yarn.test.TestScanSerialiseDeserialise.convertStringToScan(TestScanSerialiseDeserialise.java:35)
… 1 more

Reproducable: I am able to reproduce this in the sample code I have attached https://drive.google.com/file/d/0B5-H2DFQJJZeNWllejlVSjRMbDA/view?usp=sharing

Possible causes: I am suspecting that I missed supplying some dependency or there is some dependency mismatch in
underlying jars.

Appreciate any help in solving this?


Reply To: Tez and Java heap space

$
0
0

I’ve run into something similar. I have a pig job that runs perfectly fine when using MapReduce. When I use “-x tez” to run the same script, it complains about Java Heap Space. My system is configured according the recommendations in that link.

I have 10 different pig jobs with identical code using the same core data set. The only difference is each job is filtering on a different year. I”ve successfully run 2 different pig jobs using Tez. The one that fails is about 25% larger than the jobs that completed successfully. This same job works fine without using Tez. I’d love to run all of these jobs using Tez as it is about 2x faster.

hue-plugin jar file

$
0
0

I’ve downloaded the HDP 2.2 for centos 6 (linux servers have no internet access) and installed the hue-plugins rpm as instructed, but there is no jar file in the rpm:

rpm -q –filesbypkg hue-plugins-2.6.1.2.2.4.2-2.el6.x86_64
hue-plugins /usr/lib/hadoop/lib

ls -al /usr/lib/hadoop/lib
total 584
drwxr-xr-x 2 root root 4096 Mar 31 22:53 .
drwxr-xr-x 3 root root 4096 May 19 14:17 ..
-rw-r–r– 1 root root 29407 Apr 13 19:29 ambari-log4j-2.0.0.151.jar
-rw-r–r– 1 root root 551290 Apr 13 19:29 postgresql-9.1-901-1.jdbc4.jar

I was expecting a file something like hue-plugins-*.jar somewhere on the linux server.

Any ideas? Thanks.

Reply To: Falcon and schedulling

$
0
0

Hi,

if you create feed definition there is availability flag(data path or file), if this exists then only your job will be scheduled.
falcon does not scheule at specific time but on availability flag.
i have not seen schema in file, you might have to maintain it seperately and call from your program evoked by oozie workflow.

Reply To: Ambari agent installation failing

$
0
0

Hi,

I am running into similar issue. Unfortunately, ambari client seems to start only on the head node. I have copied and imported ambari commons, re-installed the agent and trying to start it. I am getting the following error:

ambari-agent start
Verifying Python version compatibility…
Using python /usr/bin/python2.6
Checking for previously running Ambari Agent…
Starting ambari-agent
Verifying ambari-agent process status…
ERROR: ambari-agent start failed. For more details, see /var/log/ambari-agent/ambari-agent.out:
====================
Traceback (most recent call last):
File “/usr/lib/python2.6/site-packages/ambari_agent/AmbariAgent.py”, line 24, in <module>
from Controller import AGENT_AUTO_RESTART_EXIT_CODE
File “/usr/lib/python2.6/site-packages/ambari_agent/Controller.py”, line 39, in <module>
from ambari_agent.Heartbeat import Heartbeat
File “/usr/lib/python2.6/site-packages/ambari_agent/Heartbeat.py”, line 28, in <module>
from ambari_agent.HostInfo import HostInfo
File “/usr/lib/python2.6/site-packages/ambari_agent/HostInfo.py”, line 35, in <module>
from resource_management.libraries.functions import packages_analyzer
ImportError: No module named resource_management.libraries.functions
====================
Agent out at: /var/log/ambari-agent/ambari-agent.out
Agent log at: /var/log/ambari-agent/ambari-agent.log

I have CentOS 6.5 with Ambari 2.0.1.

Does anybody have an idea what is going wrong?

Cheers
Andrey

Reply To: Ambari agent installation failing

$
0
0

Are you reinstalling Hadoop on existing cluster ?

Viewing all 3435 articles
Browse latest View live




Latest Images