# download
wget https://downloads.apache.org/hadoop/common/hadoop-3.2.2/hadoop-3.2.2.tar.gz
# 압축 해제
sudo tar -zxvf hadoop-3.2.2.tar.gz -C /opt/
# symbolic link
sudo ln -s /opt/hadoop-3.2.2 /opt/hadoop
# chown
sudo chown -R deploy. /opt/hadoop/
Step 2. MySQL 구성
# install mysql-server
sudo apt-get install mysql-server
# mysql 설정 작업
$ sudo vi /etc/mysql/mysql.conf.d/mysqld.cnf
port = 59306
...
#bind-address = 127.0.0.1
# restart
sudo service mysql restart
# check
$ netstat -anp | grep LISTEN
(Not all processes could be identified, non-owned process info
will not be shown, you would have to be root to see it all.)
tcp 0 0 0.0.0.0:199 0.0.0.0:* LISTEN -
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN -
tcp6 0 0 :::59306 :::* LISTEN -
tcp6 0 0 :::22 :::* LISTEN -
# 계정 작업
create user 'hive'@'localhost' identified by 'hive';
create user 'hive'@'%' identified by 'hive';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'localhost' IDENTIFIED BY 'hive';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'%' IDENTIFIED BY 'hive';
flush privileges;
Step 3. Metastore binary
Hive3 부터는 standalone 바이너리를 따로 제공해주어 이것을 이용하여 설치를 진행하였습니다.
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- These are default values meant to allow easy smoke testing of the metastore. You will
likely need to add a number of new values. -->
<configuration>
<property>
<name>metastore.thrift.uris</name>
<value>thrift://localhost:9083</value>
<description>Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore.</description>
</property>
<property>
<name>metastore.task.threads.always</name>
<value>org.apache.hadoop.hive.metastore.events.EventCleanerTask</value>
</property>
<property>
<name>metastore.expression.proxy</name>
<value>org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy</value>
</property>
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://story-hive-metastore-test-02:59306/hive?createDatabaseIfNotExist=true</value>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.cj.jdbc.Driver</value>
</property>
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>hive</value>
</property>
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>hive</value>
</property>
</configuration>
Exception in thread "main" java.lang.NoSuchMethodError: com.google.common.base.Preconditions.checkArgument(ZLjava/lang/String;Ljava/lang/Object;)V
at org.apache.hadoop.conf.Configuration.set(Configuration.java:1357)
at org.apache.hadoop.conf.Configuration.set(Configuration.java:1338)
at org.apache.hadoop.hive.metastore.conf.MetastoreConf.newMetastoreConf(MetastoreConf.java:1179)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.<init>(MetastoreSchemaTool.java:104)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.run(MetastoreSchemaTool.java:1222)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.main(MetastoreSchemaTool.java:1178)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.RunJar.run(RunJar.java:323)
at org.apache.hadoop.util.RunJar.main(RunJar.java:236)
문제
metastore와 hadoop common의 guava 버전이 다름
$ ls -tal /opt/metastore/lib/*guava*
-rw-r--r-- 1 deploy deploy 2308517 Sep 26 2018 /opt/metastore/lib/guava-19.0.jar
$ ls -tal /opt/hadoop/share/hadoop/common/lib/*guava*
-rw-r--r-- 1 deploy deploy 2747878 Jan 3 2021 /opt/hadoop/share/hadoop/common/lib/guava-27.0-jre.jar
-rw-r--r-- 1 deploy deploy 2199 Jan 3 2021 /opt/hadoop/share/hadoop/common/lib/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
....
511/511 -- Dump completed on 2012-08-23 0:56:31
Closing: com.mysql.cj.jdbc.ConnectionImpl
sqlline version 1.3.0
Initialization script completed
schemaTool completed
2021-12-22 10:06:16,465 shutdown-hook-0 INFO Log4j appears to be running in a Servlet environment, but there's no log4j-web module available. If you want better web container support, please add the log4j-web JAR to your web archive or server lib directory.
2021-12-22 10:06:16,469 shutdown-hook-0 INFO Log4j appears to be running in a Servlet environment, but there's no log4j-web module available. If you want better web container support, please add the log4j-web JAR to your web archive or server lib directory.
2021-12-22 10:06:16,478 shutdown-hook-0 WARN Unable to register Log4j shutdown hook because JVM is shutting down. Using SimpleLogger
# MySQL 확인
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| hive |
| mysql |
| performance_schema |
| sys |
+--------------------+
5 rows in set (0.00 sec)
$ cat nohup.out
2021-12-23 02:48:24: Starting Metastore Server
2021-12-23 02:48:27,842 main INFO Log4j appears to be running in a Servlet environment, but there's no log4j-web module available. If you want better web container support, please add the log4j-web JAR to your web archive or server lib directory.
$ netstat -anp | grep LISTEN
(Not all processes could be identified, non-owned process info
will not be shown, you would have to be root to see it all.)
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN -
tcp 0 0 0.0.0.0:199 0.0.0.0:* LISTEN -
tcp6 0 0 :::22 :::* LISTEN -
tcp6 0 0 :::9083 :::* LISTEN 3256/java
# 스키마 초기화 단계에서 발생한 Warning
특정 테이블이 Deprecated 되어 아래와 같은 워닝 메시지가 발생하였습니다. 참고용으로 남겨둡니다.
Warning: Changing sql mode 'NO_AUTO_CREATE_USER' is deprecated. It will be removed in a future release. (state=HY000,code=3090)
java.sql.SQLWarning: Changing sql mode 'NO_AUTO_CREATE_USER' is deprecated. It will be removed in a future release.
at com.mysql.cj.protocol.a.NativeProtocol.convertShowWarningsToSQLWarnings(NativeProtocol.java:2138)
at com.mysql.cj.jdbc.StatementImpl.getWarnings(StatementImpl.java:1731)
at sqlline.Commands.execute(Commands.java:849)
at sqlline.Commands.sql(Commands.java:733)
at sqlline.SqlLine.dispatch(SqlLine.java:795)
at sqlline.SqlLine.runCommands(SqlLine.java:1706)
at sqlline.Commands.run(Commands.java:1317)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at sqlline.ReflectiveCommandHandler.execute(ReflectiveCommandHandler.java:38)
at sqlline.SqlLine.dispatch(SqlLine.java:791)
at sqlline.SqlLine.initArgs(SqlLine.java:595)
at sqlline.SqlLine.begin(SqlLine.java:643)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.runSqlLine(MetastoreSchemaTool.java:1034)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.runSqlLine(MetastoreSchemaTool.java:1007)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.doInit(MetastoreSchemaTool.java:596)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.doInit(MetastoreSchemaTool.java:574)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.run(MetastoreSchemaTool.java:1273)
at org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool.main(MetastoreSchemaTool.java:1178)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.RunJar.run(RunJar.java:323)
at org.apache.hadoop.util.RunJar.main(RunJar.java:236)
