From 262dea91b9a5d534eae63f6c09acb0eae2d52162 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 13 Nov 2024 18:14:31 +0000 Subject: [PATCH 01/22] Add task Signed-off-by: Jacob Woffenden --- .../datasync-tasks.tf | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 terraform/environments/analytical-platform-ingestion/datasync-tasks.tf diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf new file mode 100644 index 00000000000..e87e5f6e6be --- /dev/null +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -0,0 +1,19 @@ +resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { + name = "dom1-hq-pgo-shared-group-sis-case-management-investigations" + source_location_arn = aws_datasync_location_smb.dom1_hq_pgo_shared_group_sis_case_management_investigations.arn + destination_location_arn = aws_datasync_location_s3.dom1_hq_pgo_shared_group_sis_case_management_investigations.arn + + task_report_config { + report_level = "SUCCESSES_AND_ERRORS" + output_type = "STANDARD" + s3_object_versioning = "INCLUDE" + + s3_destination { + bucket_access_role_arn = module.datasync_iam_role.iam_role_arn + s3_bucket_arn = module.datasync_bucket.s3_bucket_arn + subdirectory = "reports/dom1/hq/pgo/shared/group/sis-case-management/investigations/" + } + } + + tags = local.tags +} From 67fccd628540750aa322918fed5742536d85c8e3 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 13 Nov 2024 18:19:02 +0000 Subject: [PATCH 02/22] update locations in bucket Signed-off-by: Jacob Woffenden --- .../analytical-platform-ingestion/datasync-locations.tf | 4 ++-- .../analytical-platform-ingestion/datasync-tasks.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index bf7eff7a03b..907bce09625 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -12,7 +12,7 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "/dom1/data/hq/pgo/shared/group/sis-case-management/investigations/" + subdirectory = "datasync/dom1/data/hq/pgo/shared/group/sis-case-management/investigations/" s3_config { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn @@ -35,7 +35,7 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_itas" { s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "/dom1/data/hq/pgo/shared/group/sis-case-management/itas/" + subdirectory = "datasync/dom1/data/hq/pgo/shared/group/sis-case-management/itas/" s3_config { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index e87e5f6e6be..890b6172cf1 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -11,7 +11,7 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_inves s3_destination { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "reports/dom1/hq/pgo/shared/group/sis-case-management/investigations/" + subdirectory = "datasync/reports/dom1/hq/pgo/shared/group/sis-case-management/investigations/" } } From 5333080b8be37fba4ff6a6a664762dad764ced4a Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 21:41:12 +0000 Subject: [PATCH 03/22] update task s3 subdir Signed-off-by: Jacob Woffenden --- .../analytical-platform-ingestion/datasync-tasks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index 890b6172cf1..98cdb588fb3 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -11,7 +11,7 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_inves s3_destination { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "datasync/reports/dom1/hq/pgo/shared/group/sis-case-management/investigations/" + subdirectory = "/datasync/reports/dom1/hq/pgo/shared/group/sis-case-management/investigations/" } } From ef3e48b9eee11b8e3a00e092b82bedf8d02e8826 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 21:56:49 +0000 Subject: [PATCH 04/22] god speed Signed-off-by: Jacob Woffenden --- .../cloudwatch-log-group-policies.tf | 30 +++++++++++++++++++ .../cloudwatch-log-groups.tf | 20 +++++++++++++ .../datasync-tasks.tf | 1 + 3 files changed, 51 insertions(+) create mode 100644 terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf diff --git a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf new file mode 100644 index 00000000000..e8b0e52f22b --- /dev/null +++ b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf @@ -0,0 +1,30 @@ +ata "aws_iam_policy_document" "datasync_cloudwatch_logs" { + statement { + sid = "AllowDataSync" + effect = "Allow" + actions = [ + "logs:PutLogEvents", + "logs:CreateLogStream" + ] + principals { + type = "Service" + identifiers = ["datasync.amazonaws.com"] + } + resources = ["${module.datasync_task_logs.cloudwatch_log_group_arn}*"] + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = ["arn:aws:datasync:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:task/*"] + } + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.aws_caller_identity.current.account_id] + } + } +} + +resource "aws_cloudwatch_log_resource_policy" "datasync_cloudwatch_logs" { + policy_name = "datasync-cloudwatch-logs" + policy_document = data.aws_iam_policy_document.datasync_cloudwatch_logs.json +} diff --git a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf index e8bbad22e2a..7b951128398 100644 --- a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf +++ b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf @@ -18,3 +18,23 @@ module "connected_vpc_route53_resolver_logs" { name = "/aws/route53-resolver/connected-vpc" retention_in_days = 400 } + +module "connected_vpc_route53_resolver_logs" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + + source = "terraform-aws-modules/cloudwatch/aws//modules/log-group" + version = "5.6.0" + + name = "/aws/route53-resolver/connected-vpc" + retention_in_days = 400 +} + +module "datasync_task_logs" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + + source = "terraform-aws-modules/cloudwatch/aws//modules/log-group" + version = "5.6.0" + + name = "/aws/datasync/tasks" + retention_in_days = 400 +} diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index 98cdb588fb3..a55d6f6472a 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -2,6 +2,7 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_inves name = "dom1-hq-pgo-shared-group-sis-case-management-investigations" source_location_arn = aws_datasync_location_smb.dom1_hq_pgo_shared_group_sis_case_management_investigations.arn destination_location_arn = aws_datasync_location_s3.dom1_hq_pgo_shared_group_sis_case_management_investigations.arn + cloudwatch_log_group_arn = module.datasync_task_logs.cloudwatch_log_group_arn task_report_config { report_level = "SUCCESSES_AND_ERRORS" From e5aa58281b143edf4889c5df6492151761f22d90 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 21:59:37 +0000 Subject: [PATCH 05/22] fixes Signed-off-by: Jacob Woffenden --- .../cloudwatch-log-group-policies.tf | 2 +- .../cloudwatch-log-groups.tf | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf index e8b0e52f22b..df37bd80250 100644 --- a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf +++ b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf @@ -1,4 +1,4 @@ -ata "aws_iam_policy_document" "datasync_cloudwatch_logs" { +data "aws_iam_policy_document" "datasync_cloudwatch_logs" { statement { sid = "AllowDataSync" effect = "Allow" diff --git a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf index 7b951128398..849a53e325d 100644 --- a/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf +++ b/terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf @@ -19,16 +19,6 @@ module "connected_vpc_route53_resolver_logs" { retention_in_days = 400 } -module "connected_vpc_route53_resolver_logs" { - #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions - - source = "terraform-aws-modules/cloudwatch/aws//modules/log-group" - version = "5.6.0" - - name = "/aws/route53-resolver/connected-vpc" - retention_in_days = 400 -} - module "datasync_task_logs" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions From 0097e3eae1c0304ffabe82b5c11335661b2cc378 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 22:23:40 +0000 Subject: [PATCH 06/22] Update S3 subdirectory Signed-off-by: Jacob Woffenden --- .../analytical-platform-ingestion/datasync-locations.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index 907bce09625..983b2b4b6df 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -12,7 +12,7 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "datasync/dom1/data/hq/pgo/shared/group/sis-case-management/investigations/" + subdirectory = "/" s3_config { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn @@ -35,7 +35,7 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_itas" { s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "datasync/dom1/data/hq/pgo/shared/group/sis-case-management/itas/" + subdirectory = "/" s3_config { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn From d5c2c0b66cfbe4099b3e1395eeb6efdd6d9af48d Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 22:38:22 +0000 Subject: [PATCH 07/22] update kms Signed-off-by: Jacob Woffenden --- .../environments/analytical-platform-ingestion/iam-policies.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-ingestion/iam-policies.tf b/terraform/environments/analytical-platform-ingestion/iam-policies.tf index 885365cb44a..175606d7c4f 100644 --- a/terraform/environments/analytical-platform-ingestion/iam-policies.tf +++ b/terraform/environments/analytical-platform-ingestion/iam-policies.tf @@ -35,7 +35,7 @@ data "aws_iam_policy_document" "datasync" { "kms:DescribeKey", "kms:Decrypt", ] - resources = [module.transfer_logs_kms.key_arn] + resources = [module.s3_datasync_kms.key_arn] } statement { sid = "AllowS3BucketActions" From 4132a48f7d665eaf33b5873e8b992e4267f2f8bb Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 22:55:56 +0000 Subject: [PATCH 08/22] Add egress SMB Remove S3 replication Signed-off-by: Jacob Woffenden --- .../analytical-platform-ingestion/s3.tf | 80 +++++++++---------- .../security-groups.tf | 14 +++- 2 files changed, 52 insertions(+), 42 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/s3.tf b/terraform/environments/analytical-platform-ingestion/s3.tf index bfeff2602fb..f7dfde655ce 100644 --- a/terraform/environments/analytical-platform-ingestion/s3.tf +++ b/terraform/environments/analytical-platform-ingestion/s3.tf @@ -172,46 +172,46 @@ module "datasync_bucket" { force_destroy = true - versioning = { - enabled = true - } - - replication_configuration = { - role = module.datasync_replication_iam_role.iam_role_arn - rules = [ - { - id = "datasync-replication" - status = "Enabled" - delete_marker_replication = true - - source_selection_criteria = { - sse_kms_encrypted_objects = { - enabled = true - } - } - - destination = { - account_id = local.environment_management.account_ids["analytical-platform-data-production"] - bucket = "arn:aws:s3:::${local.environment_configuration.datasync_target_buckets[0]}" - storage_class = "STANDARD" - access_control_translation = { - owner = "Destination" - } - encryption_configuration = { - replica_kms_key_id = local.environment_configuration.mojap_land_kms_key - } - metrics = { - status = "Enabled" - minutes = 15 - } - replication_time = { - status = "Enabled" - minutes = 15 - } - } - } - ] - } + # versioning = { + # enabled = true + # } + + # replication_configuration = { + # role = module.datasync_replication_iam_role.iam_role_arn + # rules = [ + # { + # id = "datasync-replication" + # status = "Enabled" + # delete_marker_replication = true + + # source_selection_criteria = { + # sse_kms_encrypted_objects = { + # enabled = true + # } + # } + + # destination = { + # account_id = local.environment_management.account_ids["analytical-platform-data-production"] + # bucket = "arn:aws:s3:::${local.environment_configuration.datasync_target_buckets[0]}" + # storage_class = "STANDARD" + # access_control_translation = { + # owner = "Destination" + # } + # encryption_configuration = { + # replica_kms_key_id = local.environment_configuration.mojap_land_kms_key + # } + # metrics = { + # status = "Enabled" + # minutes = 15 + # } + # replication_time = { + # status = "Enabled" + # minutes = 15 + # } + # } + # } + # ] + # } server_side_encryption_configuration = { rule = { diff --git a/terraform/environments/analytical-platform-ingestion/security-groups.tf b/terraform/environments/analytical-platform-ingestion/security-groups.tf index a3539ac2d2e..9799b63ee79 100644 --- a/terraform/environments/analytical-platform-ingestion/security-groups.tf +++ b/terraform/environments/analytical-platform-ingestion/security-groups.tf @@ -90,7 +90,7 @@ module "datasync_activation_nlb_security_group" { vpc_id = module.connected_vpc.vpc_id egress_cidr_blocks = ["${local.environment_configuration.datasync_instance_private_ip}/32"] - egress_rules = ["http-80-tcp", ] + egress_rules = ["http-80-tcp"] ingress_cidr_blocks = ["${data.external.external_ip.result["ip"]}/32"] ingress_rules = ["http-80-tcp"] @@ -110,7 +110,7 @@ module "datasync_vpc_endpoint_security_group" { vpc_id = module.connected_vpc.vpc_id egress_cidr_blocks = [module.connected_vpc.vpc_cidr_block] - egress_rules = ["all-all", ] + egress_rules = ["all-all"] ingress_with_cidr_blocks = [ { @@ -160,6 +160,16 @@ module "datasync_instance_security_group" { vpc_id = module.connected_vpc.vpc_id + ingress_with_cidr_blocks = [ + { + from_port = 445 + to_port = 445 + protocol = "tcp" + description = "SMB" + cidr_blocks = "10.0.0.0/8" + } + ] + egress_with_source_security_group_id = [ { from_port = 1024 From 7022da1ad1d187f93125abf102f9113500da6b8c Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 23:05:07 +0000 Subject: [PATCH 09/22] port subdirectory diff Signed-off-by: Jacob Woffenden --- .../analytical-platform-ingestion/datasync-tasks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index a55d6f6472a..377bf3228c7 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -12,7 +12,7 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_inves s3_destination { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "/datasync/reports/dom1/hq/pgo/shared/group/sis-case-management/investigations/" + subdirectory = "datasync/reports/dom1/hq/pgo/shared/group/sis-case-management/investigations/" } } From 97970bc5a5c7574e8b034998e4f14586e753c441 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 23:10:33 +0000 Subject: [PATCH 10/22] port diff Signed-off-by: Jacob Woffenden --- .../analytical-platform-ingestion/datasync-tasks.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index 377bf3228c7..0242f4adb3a 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -4,7 +4,15 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_inves destination_location_arn = aws_datasync_location_s3.dom1_hq_pgo_shared_group_sis_case_management_investigations.arn cloudwatch_log_group_arn = module.datasync_task_logs.cloudwatch_log_group_arn + options { + gid = "NONE" + uid = "NONE" + posix_permissions = "NONE" + log_level = "NONE" + } + task_report_config { + report_overrides {} report_level = "SUCCESSES_AND_ERRORS" output_type = "STANDARD" s3_object_versioning = "INCLUDE" From b84e2a3ec1f56de36b64ec58a4953bf7b29668e6 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 23:14:09 +0000 Subject: [PATCH 11/22] fix Signed-off-by: Jacob Woffenden --- .../analytical-platform-ingestion/datasync-tasks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index 0242f4adb3a..2c302701e4d 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -8,7 +8,7 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_inves gid = "NONE" uid = "NONE" posix_permissions = "NONE" - log_level = "NONE" + log_level = "TRANSFER" } task_report_config { From 305fb4e97a8c07a4a75dec6ef5ed397093740b69 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 14 Nov 2024 23:21:12 +0000 Subject: [PATCH 12/22] absolute donut Signed-off-by: Jacob Woffenden --- .../analytical-platform-ingestion/security-groups.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-ingestion/security-groups.tf b/terraform/environments/analytical-platform-ingestion/security-groups.tf index 9799b63ee79..04b400e6df9 100644 --- a/terraform/environments/analytical-platform-ingestion/security-groups.tf +++ b/terraform/environments/analytical-platform-ingestion/security-groups.tf @@ -160,7 +160,7 @@ module "datasync_instance_security_group" { vpc_id = module.connected_vpc.vpc_id - ingress_with_cidr_blocks = [ + egress_with_cidr_blocks = [ { from_port = 445 to_port = 445 From 6c3cd23134da94493e9e22c6e92dee0e78c1477e Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Fri, 6 Dec 2024 09:59:52 +0000 Subject: [PATCH 13/22] One S3 location Update task Signed-off-by: GitHub --- .../datasync-locations.tf | 31 ++++++------------- .../datasync-tasks.tf | 4 +-- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index 983b2b4b6df..e75d73b8166 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -1,16 +1,4 @@ -resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { - server_hostname = "dom1.infra.int" - subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/Investigations/" - - user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] - password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] - - agent_arns = [aws_datasync_agent.main.arn] - - tags = local.tags -} - -resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { +resource "aws_datasync_location_s3" "mojap_datasync_s3" { s3_bucket_arn = module.datasync_bucket.s3_bucket_arn subdirectory = "/" @@ -21,9 +9,9 @@ resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_managemen tags = local.tags } -resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_itas" { +resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { server_hostname = "dom1.infra.int" - subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/ITAS/" + subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/Investigations/" user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] @@ -33,13 +21,14 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme tags = local.tags } -resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_itas" { - s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "/" +resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_itas" { + server_hostname = "dom1.infra.int" + subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/ITAS/" - s3_config { - bucket_access_role_arn = module.datasync_iam_role.iam_role_arn - } + user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] + password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] + + agent_arns = [aws_datasync_agent.main.arn] tags = local.tags } diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index 2c302701e4d..aa5d80bd38a 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -1,7 +1,7 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { name = "dom1-hq-pgo-shared-group-sis-case-management-investigations" source_location_arn = aws_datasync_location_smb.dom1_hq_pgo_shared_group_sis_case_management_investigations.arn - destination_location_arn = aws_datasync_location_s3.dom1_hq_pgo_shared_group_sis_case_management_investigations.arn + destination_location_arn = aws_datasync_location_s3.mojap_datasync_s3.arn cloudwatch_log_group_arn = module.datasync_task_logs.cloudwatch_log_group_arn options { @@ -20,7 +20,7 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_inves s3_destination { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn s3_bucket_arn = module.datasync_bucket.s3_bucket_arn - subdirectory = "datasync/reports/dom1/hq/pgo/shared/group/sis-case-management/investigations/" + subdirectory = "/" } } From d6f85ae3618e7fb1d355928256804a1d7bd1070f Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 16 Dec 2024 16:21:58 +0000 Subject: [PATCH 14/22] Add Windows instance Signed-off-by: GitHub --- .../ec2-instances.tf | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/terraform/environments/analytical-platform-compute/ec2-instances.tf b/terraform/environments/analytical-platform-compute/ec2-instances.tf index 4104af0a722..0571cdc688e 100644 --- a/terraform/environments/analytical-platform-compute/ec2-instances.tf +++ b/terraform/environments/analytical-platform-compute/ec2-instances.tf @@ -33,3 +33,40 @@ module "debug_instance" { tags = local.tags } + +module "debug_instance_windows" { + #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions + + source = "terraform-aws-modules/ec2-instance/aws" + version = "5.7.1" + + name = "network-debug-windows" + ami = "ami-0a7ebe6b52b9ca13a" # Microsoft Windows Server 2025 + instance_type = "t3.micro" + subnet_id = element(module.vpc.private_subnets, 0) + vpc_security_group_ids = [module.debug_instance_security_group.security_group_id] + associate_public_ip_address = false + key_name = "datasync-windows" + + root_block_device = [ + { + encrypted = true + volume_type = "gp3" + volume_size = 8 + } + ] + + create_iam_instance_profile = true + iam_role_policies = { + SSMCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + + metadata_options = { + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + http_tokens = "required" + instance_metadata_tags = "enabled" + } + + tags = local.tags +} From 090444e538954505d0f980a463305c7f44caf964 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 16 Dec 2024 16:28:08 +0000 Subject: [PATCH 15/22] remove windows instance Signed-off-by: GitHub --- .../ec2-instances.tf | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/terraform/environments/analytical-platform-compute/ec2-instances.tf b/terraform/environments/analytical-platform-compute/ec2-instances.tf index 0571cdc688e..4104af0a722 100644 --- a/terraform/environments/analytical-platform-compute/ec2-instances.tf +++ b/terraform/environments/analytical-platform-compute/ec2-instances.tf @@ -33,40 +33,3 @@ module "debug_instance" { tags = local.tags } - -module "debug_instance_windows" { - #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions - - source = "terraform-aws-modules/ec2-instance/aws" - version = "5.7.1" - - name = "network-debug-windows" - ami = "ami-0a7ebe6b52b9ca13a" # Microsoft Windows Server 2025 - instance_type = "t3.micro" - subnet_id = element(module.vpc.private_subnets, 0) - vpc_security_group_ids = [module.debug_instance_security_group.security_group_id] - associate_public_ip_address = false - key_name = "datasync-windows" - - root_block_device = [ - { - encrypted = true - volume_type = "gp3" - volume_size = 8 - } - ] - - create_iam_instance_profile = true - iam_role_policies = { - SSMCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" - } - - metadata_options = { - http_endpoint = "enabled" - http_put_response_hop_limit = 1 - http_tokens = "required" - instance_metadata_tags = "enabled" - } - - tags = local.tags -} From efb1907f7138b0a3015932d51803988e098b5995 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 16 Dec 2024 16:30:10 +0000 Subject: [PATCH 16/22] Update locations Signed-off-by: GitHub --- .../analytical-platform-ingestion/datasync-locations.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index e75d73b8166..89f6b16e7f5 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -10,8 +10,8 @@ resource "aws_datasync_location_s3" "mojap_datasync_s3" { } resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { - server_hostname = "dom1.infra.int" - subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/Investigations/" + server_hostname = "eucw4171nas002.dom1.infra.int" + subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/Investigations/" user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] @@ -22,8 +22,8 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme } resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_itas" { - server_hostname = "dom1.infra.int" - subdirectory = "/data/hq/PGO/Shared/Group/SIS Case Management/ITAS/" + server_hostname = "eucw4171nas002.dom1.infra.int" + subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/ITAS/" user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] From b8695913701f2467af0299be760c78880bf33bd6 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 18 Dec 2024 11:52:54 +0000 Subject: [PATCH 17/22] bucket per task --- .../datasync-locations.tf | 17 +++-------------- .../datasync-tasks.tf | 10 +++++----- .../iam-policies.tf | 8 ++++---- .../analytical-platform-ingestion/s3.tf | 5 ++--- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index 89f6b16e7f5..f4c5894c863 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -1,4 +1,4 @@ -resource "aws_datasync_location_s3" "mojap_datasync_s3" { +resource "aws_datasync_location_s3" "opg_investigations" { s3_bucket_arn = module.datasync_bucket.s3_bucket_arn subdirectory = "/" @@ -9,9 +9,9 @@ resource "aws_datasync_location_s3" "mojap_datasync_s3" { tags = local.tags } -resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { +resource "aws_datasync_location_smb" "opg_investigations" { server_hostname = "eucw4171nas002.dom1.infra.int" - subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/Investigations/" + subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/Investigations/Cases/Investigation Cases" user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] @@ -21,14 +21,3 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme tags = local.tags } -resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_itas" { - server_hostname = "eucw4171nas002.dom1.infra.int" - subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/ITAS/" - - user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] - password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] - - agent_arns = [aws_datasync_agent.main.arn] - - tags = local.tags -} diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index aa5d80bd38a..a4f53f2cadb 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -1,7 +1,7 @@ -resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_investigations" { - name = "dom1-hq-pgo-shared-group-sis-case-management-investigations" - source_location_arn = aws_datasync_location_smb.dom1_hq_pgo_shared_group_sis_case_management_investigations.arn - destination_location_arn = aws_datasync_location_s3.mojap_datasync_s3.arn +resource "aws_datasync_task" "opg_investigations" { + name = "opg-investigations" + source_location_arn = aws_datasync_location_smb.opg_investigations.arn + destination_location_arn = aws_datasync_location_s3.opg_investigations.arn cloudwatch_log_group_arn = module.datasync_task_logs.cloudwatch_log_group_arn options { @@ -19,7 +19,7 @@ resource "aws_datasync_task" "dom1_hq_pgo_shared_group_sis_case_management_inves s3_destination { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn - s3_bucket_arn = module.datasync_bucket.s3_bucket_arn + s3_bucket_arn = module.datasync_opg_investigations_bucket.s3_bucket_arn subdirectory = "/" } } diff --git a/terraform/environments/analytical-platform-ingestion/iam-policies.tf b/terraform/environments/analytical-platform-ingestion/iam-policies.tf index 175606d7c4f..b7f5c588806 100644 --- a/terraform/environments/analytical-platform-ingestion/iam-policies.tf +++ b/terraform/environments/analytical-platform-ingestion/iam-policies.tf @@ -45,7 +45,7 @@ data "aws_iam_policy_document" "datasync" { "s3:ListBucket", "s3:ListBucketMultipartUploads" ] - resources = [module.datasync_bucket.s3_bucket_arn] + resources = [module.datasync_opg_investigations_bucket.s3_bucket_arn] } statement { sid = "AllowS3ObjectActions" @@ -61,7 +61,7 @@ data "aws_iam_policy_document" "datasync" { "s3:PutObject", "s3:PutObjectTagging" ] - resources = ["${module.datasync_bucket.s3_bucket_arn}/*"] + resources = ["${module.datasync_opg_investigations_bucket.s3_bucket_arn}/*"] } } @@ -116,7 +116,7 @@ data "aws_iam_policy_document" "datasync_replication" { "s3:GetReplicationConfiguration", "s3:ListBucket" ] - resources = [module.datasync_bucket.s3_bucket_arn] + resources = [module.datasync_opg_investigations_bucket.s3_bucket_arn] } statement { sid = "SourceBucketObjectPermissions" @@ -127,7 +127,7 @@ data "aws_iam_policy_document" "datasync_replication" { "s3:GetObjectVersionTagging", "s3:ObjectOwnerOverrideToBucketOwner" ] - resources = ["${module.datasync_bucket.s3_bucket_arn}/*"] + resources = ["${module.datasync_opg_investigations_bucket.s3_bucket_arn}/*"] } } diff --git a/terraform/environments/analytical-platform-ingestion/s3.tf b/terraform/environments/analytical-platform-ingestion/s3.tf index f7dfde655ce..c4cab8508b8 100644 --- a/terraform/environments/analytical-platform-ingestion/s3.tf +++ b/terraform/environments/analytical-platform-ingestion/s3.tf @@ -161,14 +161,13 @@ module "bold_egress_bucket" { } } - -module "datasync_bucket" { +module "datasync_opg_investigations_bucket" { #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions source = "terraform-aws-modules/s3-bucket/aws" version = "4.1.2" - bucket = "mojap-ingestion-${local.environment}-datasync" + bucket = "mojap-ingestion-${local.environment}-datasync-opg-investigations" force_destroy = true From 42fe2fe0093c3bfe47d391381354940b937c7c5c Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 18 Dec 2024 11:55:50 +0000 Subject: [PATCH 18/22] Update location --- .../analytical-platform-ingestion/datasync-locations.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index f4c5894c863..77d63340202 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -1,5 +1,5 @@ resource "aws_datasync_location_s3" "opg_investigations" { - s3_bucket_arn = module.datasync_bucket.s3_bucket_arn + s3_bucket_arn = module.datasync_opg_investigations_bucket.s3_bucket_arn subdirectory = "/" s3_config { From acd14de7de9a9b033b3e77fd674ba9e4d4eb7c51 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 18 Dec 2024 13:47:29 +0000 Subject: [PATCH 19/22] Add schedule for Weds at 11PM --- .../analytical-platform-ingestion/datasync-tasks.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index a4f53f2cadb..353f114952d 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -24,5 +24,9 @@ resource "aws_datasync_task" "opg_investigations" { } } + schedule { + schedule_expression = "cron(0 23 ? * WED *)" + } + tags = local.tags } From 122f7b86cf81ac2cd19f171ede9170cbe0e38b4d Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 18 Dec 2024 13:51:35 +0000 Subject: [PATCH 20/22] couple of ports --- .../analytical-platform-ingestion/datasync-locations.tf | 2 +- .../analytical-platform-ingestion/datasync-tasks.tf | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf index 77d63340202..12d32ce466e 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf @@ -11,7 +11,7 @@ resource "aws_datasync_location_s3" "opg_investigations" { resource "aws_datasync_location_smb" "opg_investigations" { server_hostname = "eucw4171nas002.dom1.infra.int" - subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/Investigations/Cases/Investigation Cases" + subdirectory = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/Investigations/Cases/Investigation Cases/" user = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"] password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"] diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index 353f114952d..a1cdd2b7264 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -20,7 +20,6 @@ resource "aws_datasync_task" "opg_investigations" { s3_destination { bucket_access_role_arn = module.datasync_iam_role.iam_role_arn s3_bucket_arn = module.datasync_opg_investigations_bucket.s3_bucket_arn - subdirectory = "/" } } From fdd8c9eebc1f7ee18c91da60e63b9c0753b3d667 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Thu, 19 Dec 2024 09:59:12 +0000 Subject: [PATCH 21/22] Update task config --- .../analytical-platform-ingestion/datasync-tasks.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index a1cdd2b7264..8a033b2be51 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -9,11 +9,12 @@ resource "aws_datasync_task" "opg_investigations" { uid = "NONE" posix_permissions = "NONE" log_level = "TRANSFER" + verify_mode = "ONLY_FILES_TRANSFERRED" } task_report_config { report_overrides {} - report_level = "SUCCESSES_AND_ERRORS" + report_level = "ERRORS_ONLY" output_type = "STANDARD" s3_object_versioning = "INCLUDE" @@ -24,7 +25,7 @@ resource "aws_datasync_task" "opg_investigations" { } schedule { - schedule_expression = "cron(0 23 ? * WED *)" + schedule_expression = "cron(0 23 ? * THU *)" } tags = local.tags From 6aa42dee68b19d7f50acdad8a1e88b121c65ff8e Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Tue, 7 Jan 2025 14:10:59 +0000 Subject: [PATCH 22/22] Comment out schedule block in datasync-tasks --- .../analytical-platform-ingestion/datasync-tasks.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf index 8a033b2be51..7858d943ac7 100644 --- a/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf +++ b/terraform/environments/analytical-platform-ingestion/datasync-tasks.tf @@ -24,9 +24,9 @@ resource "aws_datasync_task" "opg_investigations" { } } - schedule { - schedule_expression = "cron(0 23 ? * THU *)" - } + # schedule { + # schedule_expression = "cron(0 23 ? * THU *)" + # } tags = local.tags }