[SOLVED] AWS related question with terraform: Unable to SSH into EC2 hosted in private subnet

Issue

I am just starting out to learn both terraform and aws.

In this exercise, I trying to create VPC, EC2 in a private and public subnet, NAT and IG. I am able to SSH into the EC2 hosted in the public subnet and from the EC2, I can ping the EC2 in the private subnet. However, I am unable to directly SSH into the EC2 hosted in private subnet through my local machine and I am unsure why. Will be grateful if someone can enlighten me.

create-vpc-main.tf

terraform{
    backend "s3" {
      bucket = "norman-personal-s3-bucket"
      key = "lab3-create_vpc"
      region = "ap-southeast-1"
    }
}


provider "aws" {
    region = "us-east-1"
}

resource "aws_vpc" "main" {
  cidr_block = "10.0.0.0/16"
  enable_dns_support = true
  tags = {
    Name = "Lab 3 - Project VPC"
  }
}

resource "aws_security_group" "http_server_sg" {
  name = "http_server_sg"
  //vpc_id = "vpc-c49ff1be"
  vpc_id = aws_vpc.main.id

  ingress {
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = -1
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    name = "http_server_sg"
  }
}

resource "aws_subnet" "public_subnets" {
 count      = length(var.public_subnet_cidrs)
 vpc_id     = aws_vpc.main.id
 cidr_block = element(var.public_subnet_cidrs, count.index)
 availability_zone = element(var.azs, count.index)
 
 tags = {
   Name = "Lab 3 - Public Subnet"
 }
}
 
resource "aws_subnet" "private_subnets" {
 count      = length(var.private_subnet_cidrs)
 vpc_id     = aws_vpc.main.id
 cidr_block = element(var.private_subnet_cidrs, count.index)
 availability_zone = element(var.azs, count.index)
 
 tags = {
   Name = "Lab 3 - Private Subnet" 
 }
}

resource "aws_internet_gateway" "gw" {
 vpc_id = aws_vpc.main.id
 
 tags = {
   Name = "Lab 3 - Project VPC IG"
 }
}

resource "aws_route_table" "second_rt" {
 vpc_id = aws_vpc.main.id
 
 route {
   cidr_block = "0.0.0.0/0"
   gateway_id = aws_internet_gateway.gw.id
 }
 
 tags = {
   Name = "Lab 3 - 2nd Route Table"
 }
}

resource "aws_route_table_association" "public_subnet_asso" {
 count = length(var.public_subnet_cidrs)
 subnet_id      = element(aws_subnet.public_subnets[*].id, count.index)
 route_table_id = aws_route_table.second_rt.id
}

resource "aws_eip" "nat_gateway" {
  vpc = true
}

resource "aws_nat_gateway" "nat_gateway" {
  allocation_id = aws_eip.nat_gateway.id
  subnet_id = "${element(aws_subnet.public_subnets.*.id, 0)}"
  tags = {
    "Name" = "Lab 3 - NG"
  }
}

resource "aws_route_table" "third_rt" {
  vpc_id = aws_vpc.main.id
  route {
    cidr_block = "0.0.0.0/0"
    nat_gateway_id = aws_nat_gateway.nat_gateway.id
  }
}

resource "aws_route_table_association" "private_subnet_asso" {

  count = length(var.public_subnet_cidrs)

  subnet_id = "${element(aws_subnet.private_subnets.*.id, count.index)}"
  route_table_id = aws_route_table.third_rt.id
}

create-ec2-main.tf

data "aws_vpc" "vpc" {
  filter {
    name = "tag:Name"
    values = ["Lab 3 - Project VPC"]
  }
}

data "aws_subnets" "private_subnets" {

  filter{
    name ="vpc-id"
    values = [data.aws_vpc.vpc.id]
  }

  tags = {
    Name = "Lab 3 - Private Subnet"
  }
}

data "aws_subnet" "private_subnet" {
  for_each = toset(data.aws_subnets.private_subnets.ids)
  id       = each.value
}

data "aws_subnets" "public_subnets" {

  filter{
    name ="vpc-id"
    values = [data.aws_vpc.vpc.id]
  }

  tags = {
    Name = "Lab 3 - Public Subnet"
  }
}

data "aws_subnet" "public_subnet" {
  for_each = toset(data.aws_subnets.public_subnets.ids)
  id       = each.value
}

data "aws_security_groups" "sg" {

  filter {
    name   = "vpc-id"
    values = [data.aws_vpc.vpc.id]
  }
}

data "aws_ami" "aws_linux_2_latest" {
  most_recent = true
  owners      = ["amazon"]
  filter {
    name   = "name"
    values = ["amzn2-ami-hvm-*"]
  }
}

data "aws_ami_ids" "aws_linux_2_latest_ids" {
  owners = ["amazon"]
}

terraform{
    backend "s3" {
      bucket = "norman-personal-s3-bucket"
      key = "lab3-create_ec2"
      region = "ap-southeast-1"
    }
}


provider "aws" {
    region = "us-east-1"
}



resource "aws_instance" "http_server" {

  count = length(data.aws_subnets.private_subnets.ids)
  #ami                   = "ami-062f7200baf2fa504"
  ami                    = data.aws_ami.aws_linux_2_latest.id
  key_name               = "default-ec2"
  instance_type          = "t2.micro"
  vpc_security_group_ids = data.aws_security_groups.sg.ids

  //subnet_id              = "subnet-3f7b2563"
  subnet_id = element(data.aws_subnets.private_subnets.ids,count.index)

  connection {
    type        = "ssh"
    host        = self.public_ip
    user        = "ec2-user"
    private_key = file(var.aws_key_pair)
  }

  tags = {
    Name = "Lab 3 - Private EC2 ${count.index + 1}"
  }

  # provisioner "remote-exec" {
  #   inline = [
  #     "sudo yum install httpd -y",
  #     "sudo service httpd start",
  #     "echo Welcome to in28minutes - Virtual Server is at ${self.public_dns} | sudo tee /var/www/html/index.html"
  #   ]
  # }
}

resource "aws_instance" "public_http_server" {

  count = length(data.aws_subnets.public_subnets.ids)
  #ami                   = "ami-062f7200baf2fa504"
  ami                    = data.aws_ami.aws_linux_2_latest.id
  key_name               = "default-ec2"
  instance_type          = "t2.micro"
  vpc_security_group_ids = data.aws_security_groups.sg.ids

  //subnet_id              = "subnet-3f7b2563"
  subnet_id = element(data.aws_subnets.public_subnets.ids,count.index)

  connection {
    type        = "ssh"
    host        = self.public_ip
    user        = "ec2-user"
    private_key = file(var.aws_key_pair)
  }

  tags = {
    Name = "Lab 3 - Public EC2 ${count.index + 1}"
  }

  # provisioner "remote-exec" {
  #   inline = [
  #     "sudo yum install httpd -y",
  #     "sudo service httpd start",
  #     "echo Welcome to in28minutes - Virtual Server is at ${self.public_dns} | sudo tee /var/www/html/index.html"
  #   ]
  # }
}

resource "aws_eip" "eip" {
  count = length(aws_instance.http_server)

  domain = "vpc"
  instance = aws_instance.http_server[count.index].id
}

resource "aws_eip_association" "eip_assoc" {

  count = length(aws_instance.http_server)

  instance_id   = aws_instance.http_server[count.index].id
  allocation_id = aws_eip.eip[count.index].id
}

resource "aws_eip" "public_eip" {
  count = length(aws_instance.public_http_server)

  domain = "vpc"
  instance = aws_instance.public_http_server[count.index].id
}

resource "aws_eip_association" "public_eip_assoc" {

  count = length(aws_instance.public_http_server)

  instance_id   = aws_instance.public_http_server[count.index].id
  allocation_id = aws_eip.public_eip[count.index].id
}

Solution

I am unable to directly SSH into the EC2 hosted in private subnet through my local machine and I am unsure why.

An instance in a private subnet does not have a public IP address on the Internet. There is no direct network connection between your computer on the Internet, and the server in the private VPC subnet. That's the whole purpose of making it private. It prevents the server from being directly exposed on the Internet.

If you want to SSH into the private EC2 instance, you first have to SSH into the public ECS instance, and then from that server SSH into the private EC2 instance. This concept is called a Bastion Host.

Alternatively, you can use AWS SSM session manager to create a terminal session on your private EC2 instance from your local computer, without having to jump through another computer (you are jumping through the AWS SSM service instead). This approach has the added security benefit of checking your AWS IAM permissions before allowing the connection.

Answered By - Mark B

Answer Checked By - Dawn Plyler (WPSolving Volunteer)