3

.env.dev

@@ -23,5 +23,5 @@ EXPERIMENTS_DB=experiments
 EXPERIMENTS_TRACKING_URI="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${EXPERIMENTS_DB}"
 
 # Alerts
-SMTP_EMAIL=
-SMTP_PASSWORD=
+SMTP_EMAIL=$SMTP_EMAIL
+SMTP_PASSWORD=$SMTP_PASSWORD

.gitignore

@@ -7,6 +7,8 @@
 .mage_temp_profiles
 .preferences.yaml
 .ssh_tunnel
+.terraform
+.terraform.*
 .variables/
 __pycache__/
 docker-compose.override.yml
@@ -15,4 +17,6 @@ mage-ai.db
 mage_data/
 mlruns
 secrets/
+terraform.tfstate
+terraform.tfstate.backup
 titanic_clean.csv

Dockerfile

@@ -19,6 +19,6 @@ ENV USER_CODE_PATH=${USER_CODE_PATH}
 # Install custom Python libraries
 RUN pip3 install -r ${USER_CODE_PATH}/requirements.txt
 
-ENV PYTHONPATH="${PYTHONPATH}:/home/mage_code"
+ENV PYTHONPATH="${PYTHONPATH}:${MAGE_CODE_PATH}/${PROJECT_NAME}"
 
 CMD ["/bin/sh", "-c", "/app/run_app.sh"]

mlops/metadata.yaml

@@ -16,3 +16,15 @@ features:
   polars: true
 project_uuid: 36404d0ffc214b8a89f598f3522c1a20
 help_improve_mage: true
+notification_config:
+  alert_on:
+    - trigger_failure
+    - trigger_passed_sla
+    - trigger_success
+  email_config:
+    smtp_host: smtp.gmail.com
+    smtp_user: "{{ env_var('SMTP_EMAIL') }}"
+    smtp_password: "{{ env_var('SMTP_PASSWORD') }}"
+    smtp_mail_from: "{{ env_var('SMTP_EMAIL') }}"
+    to_emails:
+      - "{{ env_var('SMTP_EMAIL') }}"

mlops/unit_5_deploying/aws/alb.tf

@@ -0,0 +1,82 @@
+# alb.tf | Load Balancer Configuration
+
+resource "aws_alb" "application_load_balancer" {
+  name               = "${var.app_name}-${var.app_environment}-alb"
+  internal           = false
+  load_balancer_type = "application"
+  subnets            = aws_subnet.public.*.id
+  security_groups    = [aws_security_group.load_balancer_security_group.id]
+
+  tags = {
+    Name        = "${var.app_name}-alb"
+    Environment = var.app_environment
+  }
+}
+
+data "http" "myip" {
+  url = "http://ipv4.icanhazip.com"
+}
+
+resource "aws_security_group" "load_balancer_security_group" {
+  vpc_id = aws_vpc.aws-vpc.id
+
+  ingress {
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["${chomp(data.http.myip.response_body)}/32"]
+  }
+
+  ingress {
+    from_port   = 80
+    to_port     = 80
+    protocol    = "tcp"
+    cidr_blocks = ["${chomp(data.http.myip.response_body)}/32"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+  tags = {
+    Name        = "${var.app_name}-sg"
+    Environment = var.app_environment
+  }
+}
+
+resource "aws_lb_target_group" "target_group" {
+  name        = "${var.app_name}-${var.app_environment}-tg"
+  port        = 6789
+  protocol    = "HTTP"
+  target_type = "ip"
+  vpc_id      = aws_vpc.aws-vpc.id
+
+  health_check {
+    healthy_threshold   = "3"
+    interval            = "30"
+    protocol            = "HTTP"
+    matcher             = "200"
+    timeout             = "5"
+    path                = "/api/status"
+    unhealthy_threshold = "2"
+  }
+
+  tags = {
+    Name        = "${var.app_name}-lb-tg"
+    Environment = var.app_environment
+  }
+}
+
+resource "aws_lb_listener" "listener" {
+  load_balancer_arn = aws_alb.application_load_balancer.id
+  port              = "80"
+  protocol          = "HTTP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.target_group.id
+  }
+}

mlops/unit_5_deploying/aws/db.tf

@@ -0,0 +1,83 @@
+# db.tf | Database Configuration
+
+resource "aws_db_subnet_group" "rds_subnet_group" {
+  name        = "${var.app_name}-${var.app_environment}-rds-subnet-group"
+  description = "${var.app_name} RDS subnet group"
+  subnet_ids  = aws_subnet.public.*.id
+  tags = {
+    Environment = var.app_environment
+  }
+}
+
+
+resource "aws_security_group" "rds_sg" {
+  name        = "${var.app_name}-${var.app_environment}-rds-sg"
+  description = "${var.app_name} RDS Security Group"
+  vpc_id      = aws_vpc.aws-vpc.id
+
+  tags = {
+    Name        = "${var.app_name}-${var.app_environment}-rds-sg"
+    Environment = var.app_environment
+  }
+
+  // allows traffic from the SG itself
+  ingress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+    self      = true
+  }
+
+  //allow traffic for TCP 5432
+  ingress {
+    from_port       = 5432
+    to_port         = 5432
+    protocol        = "tcp"
+    security_groups = ["${aws_security_group.service_security_group.id}"]
+  }
+
+  // outbound internet access
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_db_instance" "rds" {
+  identifier             = "${var.app_name}-${var.app_environment}-db"
+  allocated_storage      = 20
+  engine                 = "postgres"
+  engine_version         = "16.3"
+  instance_class         = "db.t3.micro"
+  multi_az               = false
+  db_name                = "mage"
+  username               = var.database_user     // export TF_VAR_database_username="..."
+  password               = var.database_password // export TF_VAR_database_password="..."
+  db_subnet_group_name   = aws_db_subnet_group.rds_subnet_group.id
+  vpc_security_group_ids = ["${aws_security_group.rds_sg.id}"]
+  skip_final_snapshot    = true
+  publicly_accessible    = true
+
+  tags = {
+    Environment = var.app_environment
+  }
+}
+
+# Extra resources specific to this project.
+
+resource "null_resource" "db_setup" {
+  depends_on = [aws_db_instance.rds]
+
+  provisioner "local-exec" {
+    command = <<EOT
+      PGPASSWORD="${var.database_password}" psql -U ${var.database_user} -h ${aws_db_instance.rds.address} -d ${aws_db_instance.rds.db_name} -c "CREATE DATABASE ${var.experiments_database_name};"
+    EOT
+  }
+
+  triggers = {
+    # Forces this resource to be recreated on any change of the RDS instance.
+    always_run = "${timestamp()}"
+  }
+}

mlops/unit_5_deploying/aws/efs.tf

@@ -0,0 +1,36 @@
+# efs.tf | Elastic File System Configuration
+
+resource "aws_efs_file_system" "file_system" {
+  encrypted         = true
+  performance_mode  = "generalPurpose"
+  throughput_mode   = "elastic"
+
+  tags = {
+    Name        = "${var.app_name}-efs"
+    Environment = var.app_environment
+  }
+}
+
+resource "aws_efs_mount_target" "mount_target" {
+  count = length(aws_subnet.public)
+  file_system_id = aws_efs_file_system.file_system.id
+  subnet_id      = aws_subnet.public[count.index].id
+  security_groups = [ aws_security_group.mount_target_security_group.id ]
+}
+
+
+resource "aws_security_group" "mount_target_security_group" {
+  vpc_id = aws_vpc.aws-vpc.id
+
+  ingress {
+    from_port        = 2049
+    to_port          = 2049
+    protocol         = "tcp"
+    security_groups  = [aws_security_group.service_security_group.id]
+  }
+
+  tags = {
+    Name        = "${var.app_name}-efs-sg"
+    Environment = var.app_environment
+  }
+}

mlops/unit_5_deploying/aws/env_vars.json

@@ -0,0 +1,34 @@
+[
+  {
+    "name": "ENV",
+    "value": "production"
+  },
+  {
+    "name": "AWS_ACCESS_KEY_ID",
+    "value": "${aws_access_key_id}"
+  },
+  {
+    "name": "AWS_SECRET_ACCESS_KEY",
+    "value": "${aws_secret_access_key}"
+  },
+  {
+    "name": "AWS_REGION_NAME",
+    "value": "${aws_region_name}"
+  },
+  {
+    "name": "MAGE_EC2_SUBNET_ID",
+    "value": "${ec2_subnet_id}"
+  },
+  {
+    "EXPERIMENTS_TRACKING_URI": "${experiments_tracking_uri}"
+  },
+  {
+    "SMTP_EMAIL": "${smtp_email}"
+  },
+  {
+    "SMTP_PASSWORD": "${smtp_password}"
+  },
+  {
+    "MAGE_PRESENTERS_DIRECTORY": "mlops/presenters"
+  }
+]

mlops/unit_5_deploying/aws/iam.tf

@@ -0,0 +1,73 @@
+# iam.tf | IAM Role Policies
+
+resource "aws_iam_role" "ecsTaskExecutionRole" {
+  name               = "${var.app_name}-execution-task-role"
+  assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
+  tags = {
+    Name        = "${var.app_name}-iam-role"
+    Environment = var.app_environment
+  }
+}
+
+data "aws_iam_policy_document" "assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ecs-tasks.amazonaws.com"]
+    }
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "ecsTaskExecutionRole_policy" {
+  role       = aws_iam_role.ecsTaskExecutionRole.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
+}
+
+
+# resource "aws_iam_role" "lambda_role" {
+#   name   = "${var.app_name}-lambda-role"
+#   assume_role_policy = <<EOF
+# {
+#  "Version": "2012-10-17",
+#  "Statement": [
+#    {
+#      "Action": "sts:AssumeRole",
+#      "Principal": {
+#        "Service": "lambda.amazonaws.com"
+#      },
+#      "Effect": "Allow",
+#      "Sid": ""
+#    }
+#  ]
+# }
+# EOF
+# }
+
+# resource "aws_iam_policy" "iam_policy_for_lambda" {
+#  name         = "${var.app_name}_policy_for_lambda_role"
+#  path         = "/"
+#  description  = "IAM Policy for managing ${var.app_name} lambda role"
+#  policy = <<EOF
+# {
+#  "Version": "2012-10-17",
+#  "Statement": [
+#    {
+#      "Action": [
+#        "logs:CreateLogGroup",
+#        "logs:CreateLogStream",
+#        "logs:PutLogEvents"
+#      ],
+#      "Resource": "arn:aws:logs:*:*:*",
+#      "Effect": "Allow"
+#    }
+#  ]
+# }
+# EOF
+# }
+
+# resource "aws_iam_role_policy_attachment" "attach_iam_policy_to_lambda_role" {
+#  role        = aws_iam_role.lambda_role.name
+#  policy_arn  = aws_iam_policy.iam_policy_for_lambda.arn
+# }

mlops/unit_5_deploying/aws/lambda.tf

@@ -0,0 +1,23 @@
+# # lambda.tf | Lambda Function Configuration
+
+# data "archive_file" "zip_the_python_code" {
+#   type        = "zip"
+#   source_dir  = "${path.module}/python/"
+#   output_path = "${path.module}/python/event_handler.zip"
+# }
+
+# resource "aws_lambda_function" "terraform_lambda_func" {
+#   filename        = "${path.module}/python/event_handler.zip"
+#   function_name   = "${var.app_name}-events"
+#   role            = aws_iam_role.lambda_role.arn
+#   handler         = "event_handler.lambda_handler"
+#   runtime         = "python3.8"
+#   depends_on      = [aws_iam_role_policy_attachment.attach_iam_policy_to_lambda_role,
+#                       aws_alb.application_load_balancer]
+
+#   environment {
+#     variables = {
+#       MAGE_API_HOST = aws_alb.application_load_balancer.dns_name,
+#     }
+#   }
+# }