Nginx + consul
Nginx + consul
功能
Nginx + consul 可以实现服务器动态下线、上线、水平扩展、限流、监控、灰度发布。
安装依赖
yum -y install gccgcc-c++ make libtool zlib zlib-devel openssl openssl-devel pcre pcre-devel
下载文件
wget http://nginx.org/download/nginx-1.9.9.tar.gz
tar zxfnginx-1.9.9.tar.gz
wget https://releases.hashicorp.com/consul/1.0.1/consul_1.0.1_linux_amd64.zip
unzipconsul_1.0.1_linux_amd64.zip
wgethttps://github.com/weibocom/nginx-upsync-module/archive/master.zip
unzip master.zip
配置nginx
groupadd nginx
useradd -g nginx-s /sbin/nologin nginx
mkdir -p /var/tmp/nginx/client/
mkdir -p/usr/local/nginx
编译安装nginx
./configure --prefix=/usr/local/nginx --user=nginx --group=nginx --with-http_ssl_module --with-http_flv_module --with-http_stub_status_module --with-http_gzip_static_module --with-http_realip_module --http-client-body-temp-path=/var/tmp/nginx/client/ --http-proxy-temp-path=/var/tmp/nginx/proxy/ --http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ --http-uwsgi-temp-path=/var/tmp/nginx/uwsgi --http-scgi-temp-path=/var/tmp/nginx/scgi --with-pcre--add-module=../nginx-upsync-module
make &&make install
nginx配置文件
vi /usr/local/nginx/conf/nginx.conf
http {
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user[$time_local] "$request" '
'$status $body_bytes_sent"$http_referer" '
'"$http_user_agent""$http_x_forwarded_for"'
'$upstream_addr $upstream_status$upstream_response_time $request_time';
access_log logs/access.log main;
sendfile on;
keepalive_timeout 65;
#gzip on;
upstream test {
upsync192.168.1.195:8500/v1/kv/upstreams/test upsync_timeout=6m upsync_interval=500msupsync_type=consul strong_dependency=off;
upsync_dump_path /usr/local/nginx/conf/servers/servers_test.conf;
include/usr/local/nginx/conf/servers/servers_test.conf;
}
server {
listen 80;
location = / {
proxy_pass http://test;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header real $upstream_addr;
}
location ~ /Content|Scripts/ {
proxy_pass http://test;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location = /upstream_show {
upstream_show;
}
location = /upstream_status {
stub_status on;
access_log off;
}
}
}
mkdir -p/usr/local/nginx/conf/servers/
安装consul
mkdir /usr/local/consul
mv consul /usr/local/consul/
启动consul
/usr/local/consul/consul agent-server -bootstrap-expect=1 -data-dir=/data/consul_data -node=one1-bind=192.168.2.7 -config-dir=/usr/local/consul/ -client 0.0.0.0 -ui
添加服务
我的命令：
curl -X PUThttp://192.168.2.7:8500/v1/kv/upstreams/test/192.168.2.7:8080
curl -X PUT -d '{"weight":2,"max_fails":2, "fail_timeout":10, "down":1}' http://192.168.2.7:8500/v1/kv/upstreams/test/192.168.2.7:8080
curl -X PUT http://ip:8500/v1/kv/upstreams/test/192.168.77.140:80
curl -X PUT http://ip:8500/v1/kv/upstreams/test/192.168.77.142:80
关闭服务
curl -X PUT -d '{"weight":2, "max_fails":2, "fail_timeout":10, "down":1}' http://ip:8500/v1/kv/upstreams/test/192.168.77.140:80
开启服务
curl -X PUT -d '{"weight":2, "max_fails":2,"fail_timeout":10, "down":0}'http://192.168.77.129:8500/v1/kv/upstreams/test/192.168.77.142:80
DELETE
curl -X DELETE http://localhost:8500/v1/kv/upstreams/test/192.168.77.142:80
启动ngin
nginx
问题：启动的时候报文件找不到，需要手动创建。
查看consul web信息
http://xxxx:8500
查看通过consul添加的后端服务
指令
upsync
upsync $consul/etcd.api.com:$port/v1/kv/upstreams/$upstream_name/[upsync_type=consul/etcd] [upsync_interval=second/minutes][upsync_timeout=second/minutes] [strong_dependency=off/on]
默认值：无，如果省略参数，默认参数是upsync_interval = 5s upsync_timeout = 6m strong_dependency = off
上下文：上游
说明：从领事/ etcd...拉上游服务器。
参数的含义是：
upsync_interval
从consul /etcd间隔时间拉服务器。
upsync_timeout
从consul /etcd请求超时拉服务器。
upsync_type
从conf服务器类型拉服务器。
strong_dependency
当strong_dependency打开的时候，每当nginx启动或重新加载时，nginx会从consul /etcd中取出服务器。
upsync_dump_path
upsync_dump_path $path
默认：/tmp/servers_$host.conf
上下文：上游
说明：将上游后端转储到$路径。
upsync_lb
upsync_lb $load_balance
默认值：round_robin/ ip_hash / hash模块
上下文：上游
说明：主要用于least_conn和hash的一致性，使用其中之一时，必须指出使用upsync_lb。
upstream_show
显示特定的上游所有后端服务器。
location /upstream_list {
upstream_show;
}
Consul_interface
数据可以从键/值存储或服务目录中获取。在第一种情况下，参数upsync_type必须是consul。例如
upsync 127.0.0.1:8500/v1/kv/upstreams/test upsync_timeout=6mupsync_interval=500ms upsync_type=consul strong_dependency=off;
在第二种情况下，它必须是consul_services
upsync 127.0.0.1:8500/v1/catalog/service/test upsync_timeout=6mupsync_interval=500ms upsync_type=consul_services strong_dependency=off;
在第三种情况下，它必须是consul_health：
upsync 127.0.0.1:8500/v1/health/service/test upsync_timeout=6mupsync_interval=500ms upsync_type=consul_health strong_dependency=off;
Nginx 检查模块
nginx_upstream_check_module需要根据nginx不同版本给nginx打patch，具体看Readme，upstream_check是tengine（http://tengine.taobao.org/documentation_cn.html）中的模块，主要作用用于upstream的健康检查。当遇到不可用的upstream时，nginx虽然会自动使用下一个upstream，但下一次还是会重试该upstream，导致无效请求增多。upstream_check检测到不可用会剔除该upsteam，该特性还是很有用的。
nginx.conf
http {
upstream test {
upsync127.0.0.1:8500/v1/kv/upstreams/test/ upsync_timeout=6m upsync_interval=500msupsync_type=consul strong_dependency=off;
upsync_dump_path/usr/local/nginx/conf/servers/servers_test.conf;
check interval=1000rise=2 fall=2 timeout=3000 type=http default_down=false;
check_http_send"HEAD / HTTP/1.0\r\n\r\n";
check_http_expect_alive http_2xx http_3xx;
}
#查看check ui
upstream bar {
server 127.0.0.1:8090weight=1 fail_timeout=10 max_fails=3;
}
server {
listen 8080;
location = /proxy_test{
proxy_passhttp://test;
}
location = /bar {
proxy_passhttp://bar;
}
location =/upstream_show {
upstream_show;
}
location =/upstream_status {
check_status;
access_log off;
}
}
}
gitclone https://github.com/weibocom/nginx-upsync-module
gitclone https://github.com/xiaokai-wang/nginx_upstream_check_module
# cdnginx源目录
#yum -yinstall patch
打补丁
先到nginx安装源文件目录：
#patch-p0 < ../目录/nginx_upstream_check_module-master/check_1.9.2+.patch
Eg:patch-p0 < ../nginx_upstream_check_module-master/check_1.9.2+.patch
我的命令：patch -p0 <../nginx_upstream_check_module-master/check_1.11.5+.patch
P0代表全路径
#./configure --prefix=/usr/local/nginx --user=nginx --group=nginx --with-http_ssl_module --with-http_flv_module --with-http_stub_status_module --with-http_gzip_static_module --with-http_realip_module --http-client-body-temp-path=/var/tmp/nginx/client/ --http-proxy-temp-path=/var/tmp/nginx/proxy/ --http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ --http-uwsgi-temp-path=/var/tmp/nginx/uwsgi --http-scgi-temp-path=/var/tmp/nginx/scgi --with-pcre--add-module=/opt/nginx_consul/ngix1.9.2/nginx-upsync-module--add-module=/opt/nginx_consul/ngix1.9.2/nginx_upstream_check_module
我的命令：
./configure --prefix=/usr/local/nginx --user=nginx --group=nginx --with-http_ssl_module --with-http_flv_module --with-http_stub_status_module --with-http_gzip_static_module --with-http_realip_module --http-client-body-temp-path=/var/tmp/nginx/client/ --http-proxy-temp-path=/var/tmp/nginx/proxy/ --http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ --http-uwsgi-temp-path=/var/tmp/nginx/uwsgi --http-scgi-temp-path=/var/tmp/nginx/scgi --with-pcre --add-module=../nginx-upsync-module --add-module=../nginx_upstream_check_module
--add-module这个是补丁包的路径。
make &&make install
/usr/local/nginx/sbin/nginx
http://192.168.1.195/upstream_status_check
http://192.168.1.195/upstream_show
Upstream name: test; Backend server count: 1
server 192.168.77.140:80 weight=1 max_fails=2 fail_timeout=10s;
Upstream name: bar; Backend server count: 1
server 192.168.1.195:80 weight=1 max_fails=3 fail_timeout=10s;
http://192.168.1.195/upstream_status
Active connections: 2
server accepts handled requests
2031 2031 2034
Reading: 0 Writing: 1 Waiting: 1
完整Nginx.conf
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local]"$request" '
'$status $body_bytes_sent"$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"'
'$upstream_addr $upstream_status$upstream_response_time $request_time';
access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
#
#proxy_cache_path /tmp/cache_backend keys_zone=cache_backend:10m;
upstream test {
server 127.0.0.1:11111;
upsync192.168.1.195:8500/v1/kv/upstreams/test upsync_timeout=6m upsync_interval=500msupsync_type=consul strong_dependency=off;
upsync_dump_path/usr/local/nginx/conf/servers/servers_test.conf;
check interval=1000 rise=2 fall=2timeout=3000 type=http default_down=false;
check_http_send "HEAD /HTTP/1.0\r\n\r\n";
check_http_expect_alive http_2xxhttp_3xx;
server 192.168.77.140:8080 backup;
}
upstream bar {
server 192.168.1.195:8090 weight=1 fail_timeout=10 max_fails=3;
}
server {
listen 80;
location = / {
proxy_pass http://test;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header real $upstream_addr;
}
location ~ /Content|Scripts/ {
proxy_pass http://test;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location = /bar {
proxy_pass http://bar;
}
location = /upstream_show {
upstream_show;
}
location = /upstream_status {
stub_status on;
access_log off;
}
location = /upstream_status_check {
check_status;
access_log off;
}
}
}
我自己的配置文件，可以正常运行
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user[$time_local] "$request" '
'$status $body_bytes_sent"$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"'
'$upstream_addr $upstream_status$upstream_response_time $request_time';
access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
#
#proxy_cache_path /tmp/cache_backendkeys_zone=cache_backend:10m;
upstream test {
server 127.0.0.1:11111;
upsync192.168.2.7:8500/v1/kv/upstreams/test upsync_timeout=6m upsync_interval=500msupsync_type=consul strong_dependency=off;
upsync_dump_path/usr/local/nginx/conf/servers/servers_test.conf;
check interval=1000 rise=2 fall=2timeout=3000 type=http default_down=false;
check_http_send "HEAD /HTTP/1.0\r\n\r\n";
check_http_expect_alive http_2xxhttp_3xx;
server 192.168.2.7:8080 backup;
}
upstream bar {
server 192.168.2.7:8080 weight=1fail_timeout=10 max_fails=3;
}
server {
listen 80;
location = / {
proxy_pass http://test;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header real $upstream_addr;
}
location ~ \.jsp$ {
proxy_pass http://test;
}
location~ \.(html|js|css|png|gif)$ {
#可以做静态分离，直接指定到对应的目录下如
root/soft/java/apache-tomcat-7.0.68/webapps/ROOT;
}
location = /bar {
proxy_pass http://bar;
}
location = /upstream_show {
upstream_show;
}
location = /upstream_status {
stub_status on;
access_log off;
}
location = /upstream_status_check {
check_status;
access_log off;
}
}
}