Merge pull request #8648 from ministryofjustice/feat/ap-ingest-datasy…

…nc-locations 🗄️ Add DataSync locations
ministryofjustice · Nov 13, 2024 · 07e2a2b · 07e2a2b
2 parents 73cc5e4 + cba9884
commit 07e2a2b
Show file tree

Hide file tree

Showing 7 changed files with 154 additions and 2 deletions.
diff --git a/terraform/environments/analytical-platform-ingestion/data.tf b/terraform/environments/analytical-platform-ingestion/data.tf
@@ -37,3 +37,7 @@ data "aws_network_interface" "datasync_vpc_endpoint" {
 data "aws_ec2_transit_gateway" "moj_tgw" {
   id = "tgw-026162f1ba39ce704"
 }
+
+data "aws_secretsmanager_secret_version" "datasync_dom1" {
+  secret_id = module.datasync_dom1_secret.secret_id
+}
diff --git a/terraform/environments/analytical-platform-ingestion/datasync-locations.tf b/terraform/environments/analytical-platform-ingestion/datasync-locations.tf
@@ -0,0 +1,45 @@
+resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_investigations" {
+  server_hostname = "dom1.infra.int"
+  subdirectory    = "/data/hq/PGO/Shared/Group/SIS Case Management/Investigations/"
+
+  user     = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"]
+  password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"]
+
+  agent_arns = [aws_datasync_agent.main.arn]
+
+  tags = local.tags
+}
+
+resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_investigations" {
+  s3_bucket_arn = module.datasync_bucket.s3_bucket_arn
+  subdirectory  = "/dom1/data/hq/pgo/shared/group/sis-case-management/investigations/"
+
+  s3_config {
+    bucket_access_role_arn = module.datasync_iam_role.iam_role_arn
+  }
+
+  tags = local.tags
+}
+
+resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_itas" {
+  server_hostname = "dom1.infra.int"
+  subdirectory    = "/data/hq/PGO/Shared/Group/SIS Case Management/ITAS/"
+
+  user     = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"]
+  password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"]
+
+  agent_arns = [aws_datasync_agent.main.arn]
+
+  tags = local.tags
+}
+
+resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_itas" {
+  s3_bucket_arn = module.datasync_bucket.s3_bucket_arn
+  subdirectory  = "/dom1/data/hq/pgo/shared/group/sis-case-management/itas/"
+
+  s3_config {
+    bucket_access_role_arn = module.datasync_iam_role.iam_role_arn
+  }
+
+  tags = local.tags
+}
diff --git a/terraform/environments/analytical-platform-ingestion/environment-configuration.tf b/terraform/environments/analytical-platform-ingestion/environment-configuration.tf
@@ -28,7 +28,8 @@ locals {
       notify_image_version   = "0.0.19"
 
       /* Target Buckets */
-      target_buckets = ["mojap-land-dev"]
+      target_buckets          = ["mojap-land-dev"]
+      datasync_target_buckets = ["mojap-land-dev"]
 
       /* Transfer Server */
       transfer_server_hostname   = "sftp.development.ingestion.analytical-platform.service.justice.gov.uk"
@@ -72,7 +73,8 @@ locals {
       notify_image_version   = "0.0.19"
 
       /* Target Buckets */
-      target_buckets = ["mojap-land"]
+      target_buckets          = ["mojap-land"]
+      datasync_target_buckets = ["mojap-land"]
 
       /* Transfer Server */
       transfer_server_hostname   = "sftp.ingestion.analytical-platform.service.justice.gov.uk"

diff --git a/terraform/environments/analytical-platform-ingestion/iam-policies.tf b/terraform/environments/analytical-platform-ingestion/iam-policies.tf
@@ -23,3 +23,55 @@ module "transfer_server_iam_policy" {
 
   policy = data.aws_iam_policy_document.transfer_server.json
 }
+
+data "aws_iam_policy_document" "datasync" {
+  statement {
+    sid    = "AllowKMS"
+    effect = "Allow"
+    actions = [
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:Encrypt",
+      "kms:DescribeKey",
+      "kms:Decrypt",
+    ]
+    resources = [module.transfer_logs_kms.key_arn]
+  }
+  statement {
+    sid    = "AllowS3BucketActions"
+    effect = "Allow"
+    actions = [
+      "s3:GetBucketLocation",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads"
+    ]
+    resources = [module.datasync_bucket.s3_bucket_arn]
+  }
+  statement {
+    sid    = "AllowS3ObjectActions"
+    effect = "Allow"
+    actions = [
+      "s3:AbortMultipartUpload",
+      "s3:DeleteObject",
+      "s3:GetObject",
+      "s3:GetObjectTagging",
+      "s3:GetObjectVersion",
+      "s3:GetObjectVersionTagging",
+      "s3:ListMultipartUploadParts",
+      "s3:PutObject",
+      "s3:PutObjectTagging"
+    ]
+    resources = ["${module.datasync_bucket.s3_bucket_arn}/*"]
+  }
+}
+
+module "datasync_iam_policy" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
+  version = "5.44.1"
+
+  name_prefix = "datasync"
+
+  policy = data.aws_iam_policy_document.datasync.json
+}
diff --git a/terraform/environments/analytical-platform-ingestion/iam-roles.tf b/terraform/environments/analytical-platform-ingestion/iam-roles.tf
@@ -16,3 +16,19 @@ module "transfer_server_iam_role" {
     "arn:aws:iam::aws:policy/service-role/AWSTransferLoggingAccess"
   ]
 }
+
+module "datasync_iam_role" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/iam/aws//modules/iam-assumable-role"
+  version = "5.44.1"
+
+  create_role = true
+
+  role_name_prefix  = "datasync"
+  role_requires_mfa = false
+
+  trusted_role_services = ["datasync.amazonaws.com"]
+
+  custom_role_policy_arns = [module.datasync_iam_policy.arn]
+}
diff --git a/terraform/environments/analytical-platform-ingestion/kms-keys.tf b/terraform/environments/analytical-platform-ingestion/kms-keys.tf
@@ -226,3 +226,16 @@ module "datasync_credentials_kms" {
 
   deletion_window_in_days = 7
 }
+
+module "s3_datasync_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/kms/aws"
+  version = "3.1.0"
+
+  aliases               = ["s3/datasync"]
+  description           = "DataSync S3 KMS Key"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+}
diff --git a/terraform/environments/analytical-platform-ingestion/s3.tf b/terraform/environments/analytical-platform-ingestion/s3.tf
@@ -160,3 +160,23 @@ module "bold_egress_bucket" {
     }
   }
 }
+
+module "datasync_bucket" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/s3-bucket/aws"
+  version = "4.1.2"
+
+  bucket = "mojap-ingestion-${local.environment}-datasync"
+
+  force_destroy = true
+
+  server_side_encryption_configuration = {
+    rule = {
+      apply_server_side_encryption_by_default = {
+        kms_master_key_id = module.s3_datasync_kms.key_arn
+        sse_algorithm     = "aws:kms"
+      }
+    }
+  }
+}