为什么ping定时从WebServer01到DBServer？

Question

我已经应用了以下Terraform模块，该模块：

1. 添加公共子网和私有子网
2. 配置IGW
3. 添加NAT网关
4. 为Web和DB服务器添加SGs

在应用这个之后，我无法从公共ec2实例到私有ec2实例ping/ssh。

不知道少了什么。

# Custom VPC
resource "aws_vpc" "MyVPC" {
  cidr_block       = "10.0.0.0/16"
  instance_tenancy = "default" # For Prod use "dedicated"

  tags = {
    Name = "MyVPC"
  }
}

# Creates "Main Route Table", "NACL" & "default Security Group"

# Create Public Subnet, Associate with our VPC, Auto assign Public IP
resource "aws_subnet" "PublicSubNet" {
  vpc_id                  = aws_vpc.MyVPC.id # Our VPC
  availability_zone       = "eu-west-2a"     # AZ within London, 1 Subnet = 1 AZ
  cidr_block              = "10.0.1.0/24"    #  Check using this later > "${cidrsubnet(data.aws_vpc.MyVPC.cidr_block, 4, 1)}"
  map_public_ip_on_launch = "true"           # Auto assign Public IP for Public Subnet
  tags = {
    Name = "PublicSubNet"
  }
}

# Create Private Subnet, Associate with our VPC
resource "aws_subnet" "PrivateSubNet" {
  vpc_id            = aws_vpc.MyVPC.id # Our VPC
  availability_zone = "eu-west-2b"     # AZ within London region, 1 Subnet = 1 AZ
  cidr_block        = "10.0.2.0/24"    #  Check using this later > "${cidrsubnet(data.aws_vpc.MyVPC.cidr_block, 4, 1)}"
  tags = {
    Name = "PrivateSubNet"
  }
}

# Only 1 IGW per VPC
resource "aws_internet_gateway" "MyIGW" {
  vpc_id = aws_vpc.MyVPC.id
  tags = {
    Name = "MyIGW"
  }
}

# New Public route table, so we can keep "default main" route table as Private. Route out to MyIGW
resource "aws_route_table" "MyPublicRouteTable" {
  vpc_id = aws_vpc.MyVPC.id # Our VPC

  route {                    # Route out IPV4
    cidr_block = "0.0.0.0/0" # IPV4 Route Out for all
    # ipv6_cidr_block = "::/0"        The parameter destinationCidrBlock cannot be used with the parameter destinationIpv6CidrBlock # IPV6 Route Out for all
    gateway_id = aws_internet_gateway.MyIGW.id # Target : Internet Gateway created earlier
  }
  route {                                           # Route out IPV6
    ipv6_cidr_block = "::/0"                        # IPV6 Route Out for all
    gateway_id      = aws_internet_gateway.MyIGW.id # Target : Internet Gateway created earlier
  }
  tags = {
    Name = "MyPublicRouteTable"
  }
}

# Associate "PublicSubNet" with the public route table created above, removes it from default main route table
resource "aws_route_table_association" "PublicSubNetnPublicRouteTable" {
  subnet_id      = aws_subnet.PublicSubNet.id
  route_table_id = aws_route_table.MyPublicRouteTable.id
}

# Create new security group "WebDMZ" for WebServer
resource "aws_security_group" "WebDMZ" {
  name        = "WebDMZ"
  description = "Allows SSH & HTTP requests"
  vpc_id      = aws_vpc.MyVPC.id # Our VPC : SGs cannot span VPC

  ingress {
    description = "Allows SSH requests for VPC: IPV4"
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]   # SSH restricted to my laptop public IP <My PUBLIC IP>/32
  }
  ingress {
    description = "Allows HTTP requests for VPC: IPV4"
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]   # You can use Load Balancer
  }
  ingress {
    description      = "Allows HTTP requests for VPC: IPV6"
    from_port        = 80
    to_port          = 80
    protocol         = "tcp"
    ipv6_cidr_blocks = ["::/0"]
  }
  egress {
    description = "Allows SSH requests for VPC: IPV4"
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]   # SSH restricted to my laptop public IP <My PUBLIC IP>/32
  }
  egress {
    description = "Allows HTTP requests for VPC: IPV4"
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  egress {
    description      = "Allows HTTP requests for VPC: IPV6"
    from_port        = 80
    to_port          = 80
    protocol         = "tcp"
    ipv6_cidr_blocks = ["::/0"]
  }
}

# Create new EC2 instance (WebServer01) in Public Subnet
# Get ami id from Console
resource "aws_instance" "WebServer01" {
  ami           = "ami-01a6e31ac994bbc09"
  instance_type = "t2.micro"
  subnet_id     = aws_subnet.PublicSubNet.id
  key_name = "MyEC2KeyPair"   # To connect using key pair
  security_groups = [aws_security_group.WebDMZ.id]    # Assign WebDMZ security group created above
  # vpc_security_group_ids = [aws_security_group.WebDMZ.id]
  tags = {
    Name = "WebServer01"
  }
}

# Create new security group "MyDBSG" for WebServer
resource "aws_security_group" "MyDBSG" {
  name        = "MyDBSG"
  description = "Allows Public WebServer to Communicate with Private DB Server"
  vpc_id      = aws_vpc.MyVPC.id # Our VPC : SGs cannot span VPC

  ingress {
    description = "Allows ICMP requests: IPV4" # For ping,communication, error reporting etc
    from_port   = -1
    to_port     = -1
    protocol    = "icmp"
    cidr_blocks = ["10.0.1.0/24"]    # Public Subnet CIDR block, can be "WebDMZ" security group id too as below
    security_groups = [aws_security_group.WebDMZ.id]        # Tried this as above was not working, but still doesn't work
  }
  ingress {
    description      = "Allows SSH requests: IPV4" # You can SSH from WebServer01 to DBServer, using DBServer private ip address and copying private key to WebServer
    from_port        = 22                          # ssh ec2-user@Private Ip Address -i MyPvKey.pem     Private Key pasted in MyPvKey.pem
    to_port          = 22                          # Not a good practise to use store private key on WebServer, instead use Bastion Host (Hardened Image, Secure) to connect to Private DB
    protocol         = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }
  ingress {
    description = "Allows HTTP requests: IPV4"
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }
  ingress {
    description      = "Allows HTTPS requests : IPV4"
    from_port        = 443
    to_port          = 443
    protocol         = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }
  ingress {
    description      = "Allows MySQL/Aurora requests"
    from_port        = 3306
    to_port          = 3306
    protocol         = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }
  egress {
    description = "Allows ICMP requests: IPV4" # For ping,communication, error reporting etc
    from_port   = -1
    to_port     = -1
    protocol    = "icmp"
    cidr_blocks = ["10.0.1.0/24"] # Public Subnet CIDR block, can be "WebDMZ" security group id too
  }
  egress {
    description      = "Allows SSH requests: IPV4" # You can SSH from WebServer01 to DBServer, using DBServer private ip address and copying private key to WebServer
    from_port        = 22                          # ssh ec2-user@Private Ip Address -i MyPvtKey.pem     Private Key pasted in MyPvKey.pem chmod 400 MyPvtKey.pem
    to_port          = 22                          # Not a good practise to use store private key on WebServer, instead use Bastion Host (Hardened Image, Secure) to connect to Private DB
    protocol         = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }
  egress {
    description = "Allows HTTP requests: IPV4"
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }
  egress {
    description      = "Allows HTTPS requests : IPV4"
    from_port        = 443
    to_port          = 443
    protocol         = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }
  egress {
    description      = "Allows MySQL/Aurora requests"
    from_port        = 3306
    to_port          = 3306
    protocol         = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }
}


# Create new EC2 instance (DBServer) in Private Subnet, Associate "MyDBSG" Security Group
resource "aws_instance" "DBServer" {
  ami           = "ami-01a6e31ac994bbc09"
  instance_type = "t2.micro"
  subnet_id     = aws_subnet.PrivateSubNet.id
  key_name = "MyEC2KeyPair"   # To connect using key pair
  security_groups = [aws_security_group.MyDBSG.id] # THIS WAS GIVING ERROR WHEN ASSOCIATING
  # vpc_security_group_ids = [aws_security_group.MyDBSG.id]
  tags = {
    Name = "DBServer"
  }
}

# Elastic IP required for NAT Gateway
resource "aws_eip" "nateip" {
  vpc = true
  tags = {
    Name = "NATEIP"
  }
}

# DBServer in private subnet cannot access internet, so add "NAT Gateway" in Public Subnet
# Add Target as NAT Gateway in default main route table. So there is route out to Internet.
# Now you can do yum update on DBServer

resource "aws_nat_gateway" "NATGW" {         # Create NAT Gateway in each AZ so in case of failure it can use other
  allocation_id = aws_eip.nateip.id          # Elastic IP allocation
  subnet_id     = aws_subnet.PublicSubNet.id # Public Subnet

  tags = {
    Name = "NATGW"
  }
}

# Main Route Table add NATGW as Target

resource "aws_default_route_table" "DefaultRouteTable" {
  default_route_table_id = aws_vpc.MyVPC.default_route_table_id

  route {
    cidr_block     = "0.0.0.0/0"              # IPV4 Route Out for all
    nat_gateway_id = aws_nat_gateway.NATGW.id # Target : NAT Gateway created above
  }

  tags = {
    Name = "DefaultRouteTable"
  }
}

为什么ping定时从WebServer01到DBServer？

Answer 1

允许粘贴在下面的“WebDMZ”安全组中的所有出站规则可以解决此问题。

egress {  # Allow allow traffic outbound, THIS WAS THE REASON YOU WAS NOT ABLE TO PING FROM WebServer to DBServer
    description = "Allows All Traffic Outbound from Web Server" 
    from_port   = 0
    to_port     = 0
    protocol    = -1
    cidr_blocks = ["0.0.0.0/0"] 
  }

Answer 2

没有特定的NACL，默认NACL是完全开放的，因此它们在这里不相关。

为此，DBServer上的安全组需要允许到DBServer的安全组或包含它的CIDR的出口。

aws_instance。DBServer使用aws_security_group。MyDBSG.

例如。WebServer01使用aws_安全组。WebDMZ。

aws_安全组的出口规则。WebDMZ如下所示：

  egress {
    description = "Allows SSH requests for VPC: IPV4"
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]   # SSH restricted to my laptop public IP <My PUBLIC IP>/32
  }
  egress {
    description = "Allows HTTP requests for VPC: IPV4"
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  egress {
    description      = "Allows HTTP requests for VPC: IPV6"
    from_port        = 80
    to_port          = 80
    protocol         = "tcp"
    ipv6_cidr_blocks = ["::/0"]
  }

这些出口规则意味着：

1. 允许SSH所有IPv4主机
2. 允许HTTP到所有IPv4和IPv6主机

ICMP未列出，因此ICMP回声请求将在aws_security_group之前被删除。WebDMZ.这应该在aws_instanceENI的VPC FlowLog中作为拒绝可见。WebServer01.

将此出口规则添加到aws_security_组。WebDMZ应该解决以下问题：

  egress {
    description      = "Allows ICMP requests: IPV4" # For ping,communication, error reporting etc
    from_port        = -1
    to_port          = -1
    protocol         = "icmp"
    cidr_blocks      = ["10.0.2.0/24"]
  }

DBServer可能不会响应ICMP，因此在进行此更改后，您可能仍然会看到超时。参考VPC流程日志将有助于确定差异。如果您在VPC流日志中看到ICMP流的ACCEPTs，那么问题是DBServer不响应ICMP。

aws_security_group没有。WebDMZ防止SSH，所以问题一定在其他地方。

aws_security_组的入口规则。MyDBSG如下所示。

  ingress {
    description = "Allows ICMP requests: IPV4" # For ping,communication, error reporting etc
    from_port   = -1
    to_port     = -1
    protocol    = "icmp"
    cidr_blocks = ["10.0.1.0/24"]    # Public Subnet CIDR block, can be "WebDMZ" security group id too as below
    security_groups = [aws_security_group.WebDMZ.id]        # Tried this as above was not working, but still doesn't work
  }
  ingress {
    description      = "Allows SSH requests: IPV4" # You can SSH from WebServer01 to DBServer, using DBServer private ip address and copying private key to WebServer
    from_port        = 22                          # ssh ec2-user@Private Ip Address -i MyPvKey.pem     Private Key pasted in MyPvKey.pem
    to_port          = 22                          # Not a good practise to use store private key on WebServer, instead use Bastion Host (Hardened Image, Secure) to connect to Private DB
    protocol         = "tcp"
    cidr_blocks = ["10.0.1.0/24"]
  }

这些出口规则意味着：

1. 允许来自公共子网的所有ICMP（10.0.1.0/24）
2. 允许SSH从公共子网（10.0.1.0/24）

SSH应该可以工作了。DBServer可能不接受SSH连接。

假设您的DBServer是10.0.2.123的，那么如果SSH没有运行，从运行ssh-v10.0.2.123的WebServer01可以看到：

debug1: Reading configuration data /etc/ssh/ssh_config
debug1: /etc/ssh/ssh_config line 47: Applying options for *
debug1: Connecting to 10.0.2.123 [10.0.2.123] port 22.
debug1: connect to address 10.0.2.123 port 22: Operation timed out
ssh: connect to host 10.0.2.123 port 22: Operation timed out

参考VPC流程日志将有助于确定差异。如果您在VPC流日志中看到SSH流的ACCEPTs，那么问题是DBServer不接受SSH连接。

由于这几年来发生了几次变化，我将向您介绍AWS自己维护的、用于创建发布到CloudWatch日志的流程日志的分步指南。

目前我推荐使用CloudWatch而不是S3，因为与使用Athena设置和查询S3相比，使用CloudWatch日志洞察进行查询非常容易。您只需选择用于流日志的CloudWatch日志流，并搜索感兴趣的IP或端口。

此示例CloudWatch Logs Insights查询将获得eni-0123456789abcdef0（不是真正的eni，使用您正在调试的实际eni ID）上最近的20次拒绝：

fields @timestamp,@message
| sort @timestamp desc
| filter @message like 'eni-0123456789abcdef0'
| filter @message like 'REJECT'
| limit 20

在VPC流量日志中，丢失的出口规则在源ENI上显示为拒绝。

在VPC流日志中，缺少的入口规则显示为目标ENI上的REJECT。

无状态数据包过滤器要求您处理一些神秘的事情，例如大多数（并非所有）操作系统使用端口32767-65535来处理TCP流中的应答流量。这是一种痛苦，它使NACL（无状态）成为一种巨大的痛苦。

有状态防火墙（如安全组）自动跟踪连接（有状态状态），因此您只需允许服务端口连接到源SG的出口规则中的目标SG或IP CIDR块，以及目标SG的入口规则中的源SG或IP CIDR块。

即使安全组（SGs）是有状态的，它们也是默认拒绝。这包括从源启动的出站流量。如果源SG不允许将流量出站到目标，则即使目标有允许流量出站的SG，也不允许流量出站。这是一个常见的误解。SG规则不是可传递的，它们需要在双方制定。

AWS的使用安全组保护您的实例视频很好地解释了这一点。

您应该使用aws_security_group_rule资源，而不是内联出口和入口规则。

关于安全组和安全组规则的说明：Terraform目前提供了一个独立的安全组规则资源（单个入口或出口规则），以及一个在线定义入口和出口规则的安全组资源。此时，您不能将具有内嵌规则的安全组与任何安全组规则资源一起使用。这样做将导致规则设置冲突，并将覆盖规则。

以这个带有内联规则的旧式aws_security_group为例：

resource "aws_security_group" "WebDMZ" {
  name        = "WebDMZ"
  description = "Allows SSH & HTTP requests"
  vpc_id      = aws_vpc.MyVPC.id

  ingress {
    description = "Allows HTTP requests for VPC: IPV4"
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]   # You can use Load Balancer
  }

  egress {
    description = "Allows SSH requests for VPC: IPV4"
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

将其替换为现代风格的aws_security_group，并为每个规则提供aws_security_group_规则资源：

resource "aws_security_group" "WebDMZ" {
  name        = "WebDMZ"
  description = "Allows SSH & HTTP requests"
  vpc_id      = aws_vpc.MyVPC.id
}

resource "aws_security_group_rule" "WebDMZ_HTTP_in" {
  security_group_id = aws_security_group.WebDMZ.id

  type        = "ingress"
  description = "Allows HTTP requests for VPC: IPV4"
  from_port   = 80
  to_port     = 80
  protocol    = "tcp"
  cidr_blocks = ["0.0.0.0/0"]
}

resource "aws_security_group_rule" "WebDMZ_SSH_out" {
  security_group_id = aws_security_group.WebDMZ.id

  type        = "egress"
  description = "Allows SSH requests for VPC: IPV4"
  from_port   = 22
  to_port     = 22
  protocol    = "tcp"
  cidr_blocks = ["0.0.0.0/0"]
}