vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=node02
vi /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
TYPE=Ethernet
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=static
IPADDR=192.168.2.102
PREFIX=24
GATEWAY=192.168.2.99
vi /etc/udev/rules.d/70-persistent-net.rules
reboot
vi nodes
node01
node02
node03
node04
#!/bin/bash
PASSWORD=hadoop
auto_ssh_copy_id() {
expect -c "set timeout -1;
spawn ssh-copy-id $1;
expect {
*(yes/no)* {send -- yes\r;exp_continue;}
*assword:* {send -- $2\r;exp_continue;}
eof {exit 0;}
}";
}
cat nodes | while read host
do
{
auto_ssh_copy_id $host $PASSWORD
}&wait
done
#!/bin/bash
cat nodes | while read host
do
{
scp -r $1 $host:$2
}&wait
done
#!/bin/bash
cat nodes | while read host
do
{
ssh $host $1
}&wait
done
vi /etc/sysctl.conf
末尾加上
vm.swappiness=10
./scp.sh /etc/sysctl.conf /etc/
./ssh.sh "sysctl -p"
vi /etc/rc.local
末尾加上(永久生效)
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
./scp.sh /etc/rc.local /etc/
reboot
或
./ssh.sh "echo never > /sys/kernel/mm/transparent_hugepage/enabled"
./ssh.sh "echo never > /sys/kernel/mm/transparent_hugepage/defrag"
vi /etc/selinux/config
SELINUX=disabled
./scp.sh /etc/selinux/config /etc/selinux/
reboot
tar -zxvf jdk-8u112-linux-x64.tar.gz -C /opt/program/
ln -s /opt/program/jdk1.8.0_112/ /opt/java
vi /etc/profile
末尾添加
export JAVA_HOME=/opt/java
export PATH=$JAVA_HOME/bin:$PATH
./scp.sh /opt/program/jdk1.8.0_112/ /opt/program/jdk1.8.0_112/
./scp.sh /etc/profile /etc/
./ssh.sh "ln -s /opt/program/jdk1.8.0_112/ /opt/java"
./ssh.sh "source /etc/profile"(此条无效，请在各节点手动执行)
echo "JAVA_HOME=/opt/java" >> /etc/environment
yum install -y mysql mysql-server mysql-devel
chkconfig mysqld on
service mysqld start
mysql
USE mysql;
UPDATE user SET Password=PASSWORD('你的密码') WHERE user='root';
FLUSH PRIVILEGES;
exit;
mysql -u root -p
你的密码
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '你的密码' WITH GRANT OPTION;
--hive数据库
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
--oozie数据库
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
--hue数据库
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
tar -zxvf cloudera-manager-el6-cm5.9.0_x86_64.tar.gz -C /opt/
mv /opt/cm-5.9.0/ /opt/program/
ln -s /opt/program/cm-5.9.0/ /opt/cm
mv CDH-5.9.0-1.cdh5.9.0.p0.23-el6.parcel CDH-5.9.0-1.cdh5.9.0.p0.23-el6.parcel.sha1 /opt/cloudera/parcel-repo/
mv manifest.json /opt/cloudera/parcel-repo/
cd /opt/cloudera/parcel-repo/
mv CDH-5.9.0-1.cdh5.9.0.p0.23-el6.parcel.sha1 CDH-5.9.0-1.cdh5.9.0.p0.23-el6.parcel.sha
vi /opt/cm/etc/cloudera-scm-agent/config.ini
server_host=master
mv mysql-connector-java-5.1.40-bin.jar /opt/cm/share/cmf/lib/
/opt/cm/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -p你的密码 --scm-host localhost scm scm scm
useradd --system --home=/opt/cm/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
./scp.sh /opt/program/cm-5.9.0/ /opt/program/cm-5.9.0/
./ssh.sh "ln -s /opt/program/cm-5.9.0/ /opt/cm"
/opt/cm/etc/init.d/cloudera-scm-server start
/opt/cm/etc/init.d/cloudera-scm-agent start
--hive数据库
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
--oozie数据库
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
--hue数据库
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
yum install -y perl
cat /etc/environment
find / -type f -name "*.sh" | xargs grep "as ALT_NAME"
./scp.sh /opt/cm/lib64/cmf/service/client/deploy-cc.sh /opt/cm/lib64/cmf/service/client/
cat /etc/environment
cp mysql-connector-java-5.1.40-bin.jar /opt/cloudera/parcels/CDH-5.9.0-1.cdh5.9.0.p0.23/lib/hive/lib/
cp mysql-connector-java-5.1.40-bin.jar /var/lib/oozie/
su hdfs
spark-submit \
--master yarn-client \
--class org.apache.spark.examples.SparkPi \
--driver-memory 512m \
--executor-memory 512m \
--executor-cores 2 \
/opt/cloudera/parcels/CDH-5.9.0-1.cdh5.9.0.p0.23/lib/spark/examples/lib/spark-examples-1.6.0-cdh5.9.0-hadoop2.6.0-cdh5.9.0.jar \
10
17/01/12 23:28:30 INFO spark.SparkContext: Running Spark version 1.6.0
17/01/12 23:28:33 INFO spark.SecurityManager: Changing view acls to: hdfs
17/01/12 23:28:33 INFO spark.SecurityManager: Changing modify acls to: hdfs
17/01/12 23:28:33 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(hdfs); users with modify permissions: Set(hdfs)
17/01/12 23:28:34 INFO util.Utils: Successfully started service 'sparkDriver' on port 38078.
17/01/12 23:28:36 INFO slf4j.Slf4jLogger: Slf4jLogger started
17/01/12 23:28:37 INFO Remoting: Starting remoting
17/01/12 23:28:37 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://[email protected]:34306]
17/01/12 23:28:37 INFO Remoting: Remoting now listens on addresses: [akka.tcp://[email protected]:34306]
17/01/12 23:28:37 INFO util.Utils: Successfully started service 'sparkDriverActorSystem' on port 34306.
17/01/12 23:28:37 INFO spark.SparkEnv: Registering MapOutputTracker
17/01/12 23:28:37 INFO spark.SparkEnv: Registering BlockManagerMaster
17/01/12 23:28:37 INFO storage.DiskBlockManager: Created local directory at /tmp/blockmgr-d9897e9d-bdd0-424a-acdb-b636ba57cd04
17/01/12 23:28:37 INFO storage.MemoryStore: MemoryStore started with capacity 265.1 MB
17/01/12 23:28:38 INFO spark.SparkEnv: Registering OutputCommitCoordinator
17/01/12 23:28:39 INFO util.Utils: Successfully started service 'SparkUI' on port 4040.
17/01/12 23:28:39 INFO ui.SparkUI: Started SparkUI at http://192.168.2.100:4040
17/01/12 23:28:39 INFO spark.SparkContext: Added JAR file:/opt/cloudera/parcels/CDH-5.9.0-1.cdh5.9.0.p0.23/lib/spark/examples/lib/spark-examples-1.6.0-cdh5.9.0-hadoop2.6.0-cdh5.9.0.jar at spark://192.168.2.100:38078/jars/spark-examples-1.6.0-cdh5.9.0-hadoop2.6.0-cdh5.9.0.jar with timestamp 1484234919364
17/01/12 23:28:40 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.2.100:8032
17/01/12 23:28:41 INFO yarn.Client: Requesting a new application from cluster with 2 NodeManagers
17/01/12 23:28:42 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (1024 MB per container)
17/01/12 23:28:42 INFO yarn.Client: Will allocate AM container, with 896 MB memory including 384 MB overhead
17/01/12 23:28:42 INFO yarn.Client: Setting up container launch context for our AM
17/01/12 23:28:42 INFO yarn.Client: Setting up the launch environment for our AM container
17/01/12 23:28:42 INFO yarn.Client: Preparing resources for our AM container
17/01/12 23:28:44 INFO yarn.Client: Uploading resource file:/tmp/spark-40750070-91a7-4a5b-ae27-1cfd733d0be8/__spark_conf__3134783970337565626.zip -> hdfs://master:8020/user/hdfs/.sparkStaging/application_1484232210824_0004/__spark_conf__3134783970337565626.zip
17/01/12 23:28:45 INFO spark.SecurityManager: Changing view acls to: hdfs
17/01/12 23:28:45 INFO spark.SecurityManager: Changing modify acls to: hdfs
17/01/12 23:28:45 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(hdfs); users with modify permissions: Set(hdfs)
17/01/12 23:28:45 INFO yarn.Client: Submitting application 4 to ResourceManager
17/01/12 23:28:46 INFO impl.YarnClientImpl: Submitted application application_1484232210824_0004
17/01/12 23:28:47 INFO yarn.Client: Application report for application_1484232210824_0004 (state: ACCEPTED)
17/01/12 23:28:47 INFO yarn.Client:
client token: N/A
diagnostics: N/A
ApplicationMaster host: N/A
ApplicationMaster RPC port: -1
queue: root.users.hdfs
start time: 1484234925930
final status: UNDEFINED
tracking URL: http://master:8088/proxy/application_1484232210824_0004/
user: hdfs
17/01/12 23:28:48 INFO yarn.Client: Application report for application_1484232210824_0004 (state: ACCEPTED)
17/01/12 23:28:49 INFO yarn.Client: Application report for application_1484232210824_0004 (state: ACCEPTED)
17/01/12 23:28:50 INFO yarn.Client: Application report for application_1484232210824_0004 (state: ACCEPTED)
17/01/12 23:28:51 INFO cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: ApplicationMaster registered as NettyRpcEndpointRef(null)
17/01/12 23:28:51 INFO yarn.Client: Application report for application_1484232210824_0004 (state: ACCEPTED)
17/01/12 23:28:51 INFO cluster.YarnClientSchedulerBackend: Add WebUI Filter. org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter, Map(PROXY_HOSTS -> master, PROXY_URI_BASES -> http://master:8088/proxy/application_1484232210824_0004), /proxy/application_1484232210824_0004
17/01/12 23:28:51 INFO ui.JettyUtils: Adding filter: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter
17/01/12 23:28:52 INFO yarn.Client: Application report for application_1484232210824_0004 (state: RUNNING)
17/01/12 23:28:52 INFO yarn.Client:
client token: N/A
diagnostics: N/A
ApplicationMaster host: 192.168.2.101
ApplicationMaster RPC port: 0
queue: root.users.hdfs
start time: 1484234925930
final status: UNDEFINED
tracking URL: http://master:8088/proxy/application_1484232210824_0004/
user: hdfs
17/01/12 23:28:52 INFO cluster.YarnClientSchedulerBackend: Application application_1484232210824_0004 has started running.
17/01/12 23:28:52 INFO util.Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 42832.
17/01/12 23:28:52 INFO netty.NettyBlockTransferService: Server created on 42832
17/01/12 23:28:52 INFO storage.BlockManager: external shuffle service port = 7337
17/01/12 23:28:52 INFO storage.BlockManagerMaster: Trying to register BlockManager
17/01/12 23:28:52 INFO storage.BlockManagerMasterEndpoint: Registering block manager 192.168.2.100:42832 with 265.1 MB RAM, BlockManagerId(driver, 192.168.2.100, 42832)
17/01/12 23:28:52 INFO storage.BlockManagerMaster: Registered BlockManager
17/01/12 23:28:53 INFO scheduler.EventLoggingListener: Logging events to hdfs://master:8020/user/spark/applicationHistory/application_1484232210824_0004
17/01/12 23:28:53 INFO cluster.YarnClientSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.8
17/01/12 23:28:55 INFO spark.SparkContext: Starting job: reduce at SparkPi.scala:36
17/01/12 23:28:55 INFO scheduler.DAGScheduler: Got job 0 (reduce at SparkPi.scala:36) with 10 output partitions
17/01/12 23:28:55 INFO scheduler.DAGScheduler: Final stage: ResultStage 0 (reduce at SparkPi.scala:36)
17/01/12 23:28:55 INFO scheduler.DAGScheduler: Parents of final stage: List()
17/01/12 23:28:55 INFO scheduler.DAGScheduler: Missing parents: List()
17/01/12 23:28:55 INFO scheduler.DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:32), which has no missing parents
17/01/12 23:28:57 INFO spark.ExecutorAllocationManager: Requesting 1 new executor because tasks are backlogged (new desired total will be 1)
17/01/12 23:28:57 INFO storage.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 1904.0 B, free 1904.0 B)
17/01/12 23:28:58 INFO storage.MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 1202.0 B, free 3.0 KB)
17/01/12 23:28:58 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.2.100:42832 (size: 1202.0 B, free: 265.1 MB)
17/01/12 23:28:58 INFO spark.SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1006
17/01/12 23:28:58 INFO spark.ExecutorAllocationManager: Requesting 2 new executors because tasks are backlogged (new desired total will be 3)
17/01/12 23:28:58 INFO scheduler.DAGScheduler: Submitting 10 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:32)
17/01/12 23:28:58 INFO cluster.YarnScheduler: Adding task set 0.0 with 10 tasks
17/01/12 23:28:59 INFO spark.ExecutorAllocationManager: Requesting 2 new executors because tasks are backlogged (new desired total will be 5)
17/01/12 23:29:09 INFO cluster.YarnClientSchedulerBackend: Registered executor NettyRpcEndpointRef(null) (node02:49339) with ID 1
17/01/12 23:29:09 INFO storage.BlockManagerMasterEndpoint: Registering block manager node02:37527 with 265.1 MB RAM, BlockManagerId(1, node02, 37527)
17/01/12 23:29:10 INFO spark.ExecutorAllocationManager: New executor 1 has registered (new total is 1)
17/01/12 23:29:11 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, node02, executor 1, partition 0,PROCESS_LOCAL, 2071 bytes)
17/01/12 23:29:11 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, node02, executor 1, partition 1,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:16 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on node02:37527 (size: 1202.0 B, free: 265.1 MB)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2, node02, executor 1, partition 2,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3, node02, executor 1, partition 3,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Starting task 4.0 in stage 0.0 (TID 4, node02, executor 1, partition 4,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Starting task 5.0 in stage 0.0 (TID 5, node02, executor 1, partition 5,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Starting task 6.0 in stage 0.0 (TID 6, node02, executor 1, partition 6,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 6402 ms on node02 (executor 1) (1/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 250 ms on node02 (executor 1) (2/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 308 ms on node02 (executor 1) (3/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 6855 ms on node02 (executor 1) (4/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Starting task 7.0 in stage 0.0 (TID 7, node02, executor 1, partition 7,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 4.0 in stage 0.0 (TID 4) in 266 ms on node02 (executor 1) (5/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 5.0 in stage 0.0 (TID 5) in 190 ms on node02 (executor 1) (6/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Starting task 8.0 in stage 0.0 (TID 8, node02, executor 1, partition 8,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 6.0 in stage 0.0 (TID 6) in 156 ms on node02 (executor 1) (7/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Starting task 9.0 in stage 0.0 (TID 9, node02, executor 1, partition 9,PROCESS_LOCAL, 2073 bytes)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 7.0 in stage 0.0 (TID 7) in 149 ms on node02 (executor 1) (8/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 8.0 in stage 0.0 (TID 8) in 131 ms on node02 (executor 1) (9/10)
17/01/12 23:29:17 INFO scheduler.TaskSetManager: Finished task 9.0 in stage 0.0 (TID 9) in 128 ms on node02 (executor 1) (10/10)
17/01/12 23:29:17 INFO scheduler.DAGScheduler: ResultStage 0 (reduce at SparkPi.scala:36) finished in 19.406 s
17/01/12 23:29:17 INFO cluster.YarnScheduler: Removed TaskSet 0.0, whose tasks have all completed, from pool
17/01/12 23:29:17 INFO scheduler.DAGScheduler: Job 0 finished: reduce at SparkPi.scala:36, took 22.241001 s
Pi is roughly 3.142676
17/01/12 23:29:18 INFO ui.SparkUI: Stopped Spark web UI at http://192.168.2.100:4040
17/01/12 23:29:18 INFO cluster.YarnClientSchedulerBackend: Interrupting monitor thread
17/01/12 23:29:18 INFO cluster.YarnClientSchedulerBackend: Shutting down all executors
17/01/12 23:29:18 INFO cluster.YarnClientSchedulerBackend: Asking each executor to shut down
17/01/12 23:29:19 INFO cluster.YarnClientSchedulerBackend: Stopped
17/01/12 23:29:19 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
17/01/12 23:29:19 INFO storage.MemoryStore: MemoryStore cleared
17/01/12 23:29:19 INFO storage.BlockManager: BlockManager stopped
17/01/12 23:29:19 INFO storage.BlockManagerMaster: BlockManagerMaster stopped
17/01/12 23:29:19 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
17/01/12 23:29:19 INFO spark.SparkContext: Successfully stopped SparkContext
17/01/12 23:29:20 INFO remote.RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.
17/01/12 23:29:20 INFO util.ShutdownHookManager: Shutdown hook called
17/01/12 23:29:20 INFO remote.RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.
17/01/12 23:29:20 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-40750070-91a7-4a5b-ae27-1cfd733d0be8
