Kafka是一种高吞吐量的分布式发布订阅消息系统，它可以处理消费者规模的网站中的所有动作流数据。 这种动作（网页浏览，搜索和其他用户的行动）是在现代网络上的许多社会功能的一个关键因素。 这些数据通常是由于吞吐量的要求而通过处理日志和日志聚合来解决。 对于像Hadoop的一样的日志数据和离线分析系统，但又要求实时处理的限制，这是一个可行的解决方案。Kafka的目的是通过Hadoop的并行加载机制来统一线上和离线的消息处理，也是为了通过集群来提供实时的消费
实验环境
centos6.9_x64
实验软件
jdk-7u80-linux-x64.tar.gz
zookeeper-3.4.6.tar.gz
kafka_2.10-0.9.0.1.tgz
软件安装
tar zxvf jdk-7u80-linux-x64.tar.gz
tar zxvf zookeeper-3.4.6.tar.gz
tar zxvf kafka_2.10-0.9.0.1.tgz
mv jdk1.7.0_80 /usr/local/java
mv zookeeper-3.4.6 /usr/local/zookeeper
mv kafka_2.10-0.9.0.1 /usr/local/kafka
vim /etc/profile
export JAVA_HOME=/usr/local/java
export PATH=$JAVA_HOME/bin:$PATH
export CLAsspATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export zookeeper_home=/usr/local/zookeeper 最后一行添加几行配置
source /etc/profile
java -version
java version "1.7.0_80"
Java(TM) SE Runtime Environment (build 1.7.0_80-b15)
Java HotSpot(TM) 64-Bit Server VM (build 24.80-b11,mixed mode)
hostname
centos6-1
vim /etc/hosts
192.168.1.104 centos6-1
cp /usr/local/zookeeper/conf/zoo_sample.cfg /usr/local/zookeeper/conf/zoo_sample.cfg.bak
sh /usr/local/zookeeper/bin/zkServer.sh start
JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
grep: /usr/local/zookeeper/bin/../conf/zoo.cfg: No such file or directory 报错因为配置文件没有从新命名
mkdir: cannot create directory `': No such file or directory
Starting zookeeper ... STARTED
cp /usr/local/kafka/config/server.properties /usr/local/kafka/config/server.properties.bak
mkdir -p /usr/local/kafka/log/kafka
sed -i 's/#port=9092/port=9092/g' /usr/local/kafka/config/server.properties
sed -i 's/#host.name=localhost/host.name=192.168.1.104/g' /usr/local/kafka/config/server.properties
sed -i 's/zookeeper.connect=localhost:2181/zookeeper.connect=192.168.1.104:2181/g' /usr/local/kafka/config/server.properties 192.168.1.104为服务器ip
vim /usr/local/kafka/config/server.properties
log.dirs=/tmp/kafka-logs 找到
log.dirs=/usr/local/kafka/log/kafka 改为
mkdir /usr/local/kafka/zookeeper
mkdir /usr/local/kafka/log/zookeeper
cp /usr/local/kafka/config/zookeeper.properties /usr/local/kafka/config/zookeeper.properties.bak
vim /usr/local/kafka/config/zookeeper.properties
dataDir=/tmp/zookeeper 找到
dataDir=/usr/local/kafka/log/zookeeper 改为
dataLogDir=/usr/local/kafka/log/zookeeper改为
clientPort=2181
maxClientCnxns=100 默认为0 改为100
tickTime=2000 添加两行配置
initLimit=10
sh /usr/local/kafka/bin/zookeeper-server-start.sh /usr/local/kafka/config/zookeeper.properties & 启动zookeeper
sh/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties & 启动kafka
mkdir /usr/local/kafka/sh
touch /usr/local/kafka/sh/kafkastart.sh
vim /usr/local/kafka/sh/kafkastart.sh
#!/bin/bash
/usr/local/kafka/bin/zookeeper-server-start.sh /usr/local/kafka/config/zookeeper.properties &
sleep 5
/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties & 写入几行配置
touch /usr/local/kafka/sh/kafkastop.sh
vim /usr/local/kafka/sh/kafkastop.sh
/usr/local/kafka/bin/zookeeper-server-stop.sh /usr/local/kafka/config/zookeeper.properties &
/usr/local/kafka/bin/kafka-server-stop.sh /usr/local/kafka/config/server.properties &
chmod +x /usr/local/kafka/sh/kafkastart.sh
chmod +x /usr/local/kafka/sh/kafkastop.sh
vim /etc/rc.d/rc.local
sh /usr/local/kafka/sh/kafkastart.sh &
sh /usr/local/kafka/sh/kafkastart.sh
sh /usr/local/kafka/sh/kafkastop.sh 添加几行配置
netstat -tuplna | grep 2181
tcp 0 0 :::2181 :::* LISTEN 1749/java
tcp 0 0 ::ffff:192.168.1.104:2181 ::ffff:192.168.1.104:33490 TIME_WAIT -
tcp 0 0 ::ffff:192.168.1.104:2181 ::ffff:192.168.1.104:33450 TIME_WAIT -
tcp 0 0 ::ffff:192.168.1.104:2181 ::ffff:192.168.1.104:33488 TIME_WAIT -
tcp 0 0 ::ffff:192.168.1.104:2181 ::ffff:192.168.1.104:33432 TIME_WAIT -
netstat -tuplna | grep 9092
tcp 0 0 :::9092 :::* LISTEN 1941/java
tcp 0 0 ::ffff:192.168.1.102:9092 ::ffff:192.168.1.102:35412 ESTABLISHED 1941/java
tcp 0 0 ::ffff:192.168.1.102:35412 ::ffff:192.168.1.102:9092 ESTABLISHED 1941/java
ps -aux | grep zookeeper
Warning: bad Syntax,perhaps a bogus '-'? See /usr/share/doc/procps-3.2.8/FAQ
root 1749 0.1 2.2 1336472 45432 pts/0 Sl 21:57 0:02 /usr/local/java/bin/java -Dzookeeper.log.dir=. -Dzookeeper.root.logger=INFO,CONSOLE -cp /usr/local/zookeeper/bin/../build/classes:/usr/local/zookeeper/bin/../build/lib/*.jar:/usr/local/zookeeper/bin/../lib/slf4j-log4j12-1.6.1.jar:/usr/local/zookeeper/bin/../lib/slf4j-api-1.6.1.jar:/usr/local/zookeeper/bin/../lib/netty-3.7.0.Final.jar:/usr/local/zookeeper/bin/../lib/log4j-1.2.16.jar:/usr/local/zookeeper/bin/../lib/jline-0.9.94.jar:/usr/local/zookeeper/bin/../zookeeper-3.4.6.jar:/usr/local/zookeeper/bin/../src/java/lib/*.jar:/usr/local/zookeeper/bin/../conf:.:/usr/local/java/lib/dt.jar:/usr/local/java/lib/tools.jar -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=false org.apache.zookeeper.server.quorum.QuorumPeerMain /usr/local/zookeeper/bin/../conf/zoo.cfg
root 2032 0.0 0.0 103328 892 pts/0 S+ 22:16 0:00 grep zookeeper
ps -aux | grep kafka
root 1941 1.4 13.1 2089928 268608 pts/1 Sl 22:13 0:03 /usr/local/java/bin/java -Xmx1G -xms1G -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+disableExplicitGC -Djava.awt.headless=true -Xloggc:/usr/local/kafka/bin/../logs/kafkaServer-gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dkafka.logs.dir=/usr/local/kafka/bin/../logs -Dlog4j.configuration=file:/usr/local/kafka/bin/../config/log4j.properties -cp .:/usr/local/java/lib/dt.jar:/usr/local/java/lib/tools.jar:/usr/local/kafka/bin/../libs/* kafka.Kafka /usr/local/kafka/config/server.properties
root 2034 0.0 0.0 103324 884 pts/0 S+ 22:17 0:00 grep kafka
jps
1941 Kafka
2035 Jps
1749 QuorumPeerMain
