diff --git a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf new file mode 100644 index 00000000000..df37bd80250 --- /dev/null +++ b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf @@ -0,0 +1,30 @@ +data "aws_iam_policy_document" "datasync_cloudwatch_logs" { + statement { + sid = "AllowDataSync" + effect = "Allow" + actions = [ + "logs:PutLogEvents", + "logs:CreateLogStream" + ] + principals { + type = "Service" + identifiers = ["datasync.amazonaws.com"] + } + resources = ["${module.datasync_task_logs.cloudwatch_log_group_arn}*"] + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = ["arn:aws:datasync:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:task/*"] + } + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + } +} + +resource "aws_cloudwatch_log_resource_policy" "datasync_cloudwatch_logs" { + policy_name = "datasync-cloudwatch-logs" + policy_document = data.aws_iam_policy_document.datasync_cloudwatch_logs.json +} diff --git a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf index e8bbad22e2a..849a53e325d 100644 --- a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf +++ b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf @@ -18,3 +18,13 @@ module "connected_vpc_route53_resolver_logs" { name = "/aws/route53-resolver/connected-vpc" retention_in_days = 400 } + +module "datasync_task_logs" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + + source = "terraform-aws-modules/cloudwatch/aws//modules/log-group" + version = "5.6.0" + + name = "/aws/datasync/tasks" + retention_in_days = 400 +} diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index bf7eff7a03b..12d32ce466e 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -1,18 +1,6 @@ -resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { - server_hostname = "dom1.infra.int" - subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/Investigations/" - - user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] - password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] - - agent_arns = [aws_datasync_agent.main.arn] - - tags = local.tags -} - -resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { - s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "/dom1/data/hq/pgo/shared/group/sis-case-management/investigations/" +resource "aws_datasync_location_s3" "opg_investigations" { + s3_bucket_arn = module.datasync_opg_investigations_bucket.s3_bucket_arn + subdirectory = "/" s3_config { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn @@ -21,9 +9,9 @@ resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_managemen tags = local.tags } -resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_itas" { - server_hostname = "dom1.infra.int" - subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/ITAS/" +resource "aws_datasync_location_smb" "opg_investigations" { + server_hostname = "eucw4171nas002.dom1.infra.int" + subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/Investigations/Cases/Investigation Cases/" user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] @@ -33,13 +21,3 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme tags = local.tags } -resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_itas" { - s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "/dom1/data/hq/pgo/shared/group/sis-case-management/itas/" - - s3_config { - bucket_access_role_arn = module.datasync_iam_role.iam_role_arn - } - - tags = local.tags -} diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf new file mode 100644 index 00000000000..7858d943ac7 --- /dev/null +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -0,0 +1,32 @@ +resource "aws_datasync_task" "opg_investigations" { + name = "opg-investigations" + source_location_arn = aws_datasync_location_smb.opg_investigations.arn + destination_location_arn = aws_datasync_location_s3.opg_investigations.arn + cloudwatch_log_group_arn = module.datasync_task_logs.cloudwatch_log_group_arn + + options { + gid = "NONE" + uid = "NONE" + posix_permissions = "NONE" + log_level = "TRANSFER" + verify_mode = "ONLY_FILES_TRANSFERRED" + } + + task_report_config { + report_overrides {} + report_level = "ERRORS_ONLY" + output_type = "STANDARD" + s3_object_versioning = "INCLUDE" + + s3_destination { + bucket_access_role_arn = module.datasync_iam_role.iam_role_arn + s3_bucket_arn = module.datasync_opg_investigations_bucket.s3_bucket_arn + } + } + + # schedule { + # schedule_expression = "cron(0 23 ? * THU *)" + # } + + tags = local.tags +} diff --git a/terraform/environments/analytical-platform-ingestion/iam-policies.tf b/terraform/environments/analytical-platform-ingestion/iam-policies.tf index 885365cb44a..b7f5c588806 100644 --- a/terraform/environments/analytical-platform-ingestion/iam-policies.tf +++ b/terraform/environments/analytical-platform-ingestion/iam-policies.tf @@ -35,7 +35,7 @@ data "aws_iam_policy_document" "datasync" { "kms:DescribeKey", "kms:Decrypt", ] - resources = [module.transfer_logs_kms.key_arn] + resources = [module.s3_datasync_kms.key_arn] } statement { sid = "AllowS3BucketActions" @@ -45,7 +45,7 @@ data "aws_iam_policy_document" "datasync" { "s3:ListBucket", "s3:ListBucketMultipartUploads" ] - resources = [module.datasync_bucket.s3_bucket_arn] + resources = [module.datasync_opg_investigations_bucket.s3_bucket_arn] } statement { sid = "AllowS3ObjectActions" @@ -61,7 +61,7 @@ data "aws_iam_policy_document" "datasync" { "s3:PutObject", "s3:PutObjectTagging" ] - resources = ["${module.datasync_bucket.s3_bucket_arn}/*"] + resources = ["${module.datasync_opg_investigations_bucket.s3_bucket_arn}/*"] } } @@ -116,7 +116,7 @@ data "aws_iam_policy_document" "datasync_replication" { "s3:GetReplicationConfiguration", "s3:ListBucket" ] - resources = [module.datasync_bucket.s3_bucket_arn] + resources = [module.datasync_opg_investigations_bucket.s3_bucket_arn] } statement { sid = "SourceBucketObjectPermissions" @@ -127,7 +127,7 @@ data "aws_iam_policy_document" "datasync_replication" { "s3:GetObjectVersionTagging", "s3:ObjectOwnerOverrideToBucketOwner" ] - resources = ["${module.datasync_bucket.s3_bucket_arn}/*"] + resources = ["${module.datasync_opg_investigations_bucket.s3_bucket_arn}/*"] } } diff --git a/terraform/environments/analytical-platform-ingestion/s3.tf b/terraform/environments/analytical-platform-ingestion/s3.tf index bfeff2602fb..c4cab8508b8 100644 --- a/terraform/environments/analytical-platform-ingestion/s3.tf +++ b/terraform/environments/analytical-platform-ingestion/s3.tf @@ -161,57 +161,56 @@ module "bold_egress_bucket" { } } - -module "datasync_bucket" { +module "datasync_opg_investigations_bucket" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions source = "terraform-aws-modules/s3-bucket/aws" version = "4.1.2" - bucket = "mojap-ingestion-${local.environment}-datasync" + bucket = "mojap-ingestion-${local.environment}-datasync-opg-investigations" force_destroy = true - versioning = { - enabled = true - } - - replication_configuration = { - role = module.datasync_replication_iam_role.iam_role_arn - rules = [ - { - id = "datasync-replication" - status = "Enabled" - delete_marker_replication = true - - source_selection_criteria = { - sse_kms_encrypted_objects = { - enabled = true - } - } - - destination = { - account_id = local.environment_management.account_ids["analytical-platform-data-production"] - bucket = "arn:aws:s3:::${local.environment_configuration.datasync_target_buckets[0]}" - storage_class = "STANDARD" - access_control_translation = { - owner = "Destination" - } - encryption_configuration = { - replica_kms_key_id = local.environment_configuration.mojap_land_kms_key - } - metrics = { - status = "Enabled" - minutes = 15 - } - replication_time = { - status = "Enabled" - minutes = 15 - } - } - } - ] - } + # versioning = { + # enabled = true + # } + + # replication_configuration = { + # role = module.datasync_replication_iam_role.iam_role_arn + # rules = [ + # { + # id = "datasync-replication" + # status = "Enabled" + # delete_marker_replication = true + + # source_selection_criteria = { + # sse_kms_encrypted_objects = { + # enabled = true + # } + # } + + # destination = { + # account_id = local.environment_management.account_ids["analytical-platform-data-production"] + # bucket = "arn:aws:s3:::${local.environment_configuration.datasync_target_buckets[0]}" + # storage_class = "STANDARD" + # access_control_translation = { + # owner = "Destination" + # } + # encryption_configuration = { + # replica_kms_key_id = local.environment_configuration.mojap_land_kms_key + # } + # metrics = { + # status = "Enabled" + # minutes = 15 + # } + # replication_time = { + # status = "Enabled" + # minutes = 15 + # } + # } + # } + # ] + # } server_side_encryption_configuration = { rule = { diff --git a/terraform/environments/analytical-platform-ingestion/security-groups.tf b/terraform/environments/analytical-platform-ingestion/security-groups.tf index a3539ac2d2e..04b400e6df9 100644 --- a/terraform/environments/analytical-platform-ingestion/security-groups.tf +++ b/terraform/environments/analytical-platform-ingestion/security-groups.tf @@ -90,7 +90,7 @@ module "datasync_activation_nlb_security_group" { vpc_id = module.connected_vpc.vpc_id egress_cidr_blocks = ["${local.environment_configuration.datasync_instance_private_ip}/32"] - egress_rules = ["http-80-tcp", ] + egress_rules = ["http-80-tcp"] ingress_cidr_blocks = ["${data.external.external_ip.result["ip"]}/32"] ingress_rules = ["http-80-tcp"] @@ -110,7 +110,7 @@ module "datasync_vpc_endpoint_security_group" { vpc_id = module.connected_vpc.vpc_id egress_cidr_blocks = [module.connected_vpc.vpc_cidr_block] - egress_rules = ["all-all", ] + egress_rules = ["all-all"] ingress_with_cidr_blocks = [ { @@ -160,6 +160,16 @@ module "datasync_instance_security_group" { vpc_id = module.connected_vpc.vpc_id + egress_with_cidr_blocks = [ + { + from_port = 445 + to_port = 445 + protocol = "tcp" + description = "SMB" + cidr_blocks = "10.0.0.0/8" + } + ] + egress_with_source_security_group_id = [ { from_port = 1024