Merge branch 'main' into maintenance/ingestion

terraform/environments/analytical-platform-ingestion/cloudwatch-log-group-policies.tf

@@ -0,0 +1,30 @@
+data "aws_iam_policy_document" "datasync_cloudwatch_logs" {
+  statement {
+    sid    = "AllowDataSync"
+    effect = "Allow"
+    actions = [
+      "logs:PutLogEvents",
+      "logs:CreateLogStream"
+    ]
+    principals {
+      type        = "Service"
+      identifiers = ["datasync.amazonaws.com"]
+    }
+    resources = ["${module.datasync_task_logs.cloudwatch_log_group_arn}*"]
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = ["arn:aws:datasync:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:task/*"]
+    }
+    condition {
+      test     = "StringEquals"
+      variable = "aws:SourceAccount"
+      values   = [data.aws_caller_identity.current.account_id]
+    }
+  }
+}
+
+resource "aws_cloudwatch_log_resource_policy" "datasync_cloudwatch_logs" {
+  policy_name     = "datasync-cloudwatch-logs"
+  policy_document = data.aws_iam_policy_document.datasync_cloudwatch_logs.json
+}

terraform/environments/analytical-platform-ingestion/cloudwatch-log-groups.tf

@@ -18,3 +18,13 @@ module "connected_vpc_route53_resolver_logs" {
   name              = "/aws/route53-resolver/connected-vpc"
   retention_in_days = 400
 }
+
+module "datasync_task_logs" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/cloudwatch/aws//modules/log-group"
+  version = "5.6.0"
+
+  name              = "/aws/datasync/tasks"
+  retention_in_days = 400
+}

terraform/environments/analytical-platform-ingestion/datasync-locations.tf

@@ -1,18 +1,6 @@
-resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_investigations" {
-  server_hostname = "dom1.infra.int"
-  subdirectory    = "/data/hq/PGO/Shared/Group/SIS Case Management/Investigations/"
-
-  user     = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"]
-  password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"]
-
-  agent_arns = [aws_datasync_agent.main.arn]
-
-  tags = local.tags
-}
-
-resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_investigations" {
-  s3_bucket_arn = module.datasync_bucket.s3_bucket_arn
-  subdirectory  = "/dom1/data/hq/pgo/shared/group/sis-case-management/investigations/"
+resource "aws_datasync_location_s3" "opg_investigations" {
+  s3_bucket_arn = module.datasync_opg_investigations_bucket.s3_bucket_arn
+  subdirectory  = "/"
 
   s3_config {
     bucket_access_role_arn = module.datasync_iam_role.iam_role_arn
@@ -21,9 +9,9 @@ resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_managemen
   tags = local.tags
 }
 
-resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_management_itas" {
-  server_hostname = "dom1.infra.int"
-  subdirectory    = "/data/hq/PGO/Shared/Group/SIS Case Management/ITAS/"
+resource "aws_datasync_location_smb" "opg_investigations" {
+  server_hostname = "eucw4171nas002.dom1.infra.int"
+  subdirectory    = "/mojshared002$/FITS_3635/Shared/Group/SIS Case Management/Investigations/Cases/Investigation Cases/"
 
   user     = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["username"]
   password = jsondecode(data.aws_secretsmanager_secret_version.datasync_dom1.secret_string)["password"]
@@ -33,13 +21,3 @@ resource "aws_datasync_location_smb" "dom1_hq_pgo_shared_group_sis_case_manageme
   tags = local.tags
 }
 
-resource "aws_datasync_location_s3" "dom1_hq_pgo_shared_group_sis_case_management_itas" {
-  s3_bucket_arn = module.datasync_bucket.s3_bucket_arn
-  subdirectory  = "/dom1/data/hq/pgo/shared/group/sis-case-management/itas/"
-
-  s3_config {
-    bucket_access_role_arn = module.datasync_iam_role.iam_role_arn
-  }
-
-  tags = local.tags
-}

terraform/environments/analytical-platform-ingestion/datasync-tasks.tf

@@ -0,0 +1,32 @@
+resource "aws_datasync_task" "opg_investigations" {
+  name                     = "opg-investigations"
+  source_location_arn      = aws_datasync_location_smb.opg_investigations.arn
+  destination_location_arn = aws_datasync_location_s3.opg_investigations.arn
+  cloudwatch_log_group_arn = module.datasync_task_logs.cloudwatch_log_group_arn
+
+  options {
+    gid               = "NONE"
+    uid               = "NONE"
+    posix_permissions = "NONE"
+    log_level         = "TRANSFER"
+    verify_mode       = "ONLY_FILES_TRANSFERRED"
+  }
+
+  task_report_config {
+    report_overrides {}
+    report_level         = "ERRORS_ONLY"
+    output_type          = "STANDARD"
+    s3_object_versioning = "INCLUDE"
+
+    s3_destination {
+      bucket_access_role_arn = module.datasync_iam_role.iam_role_arn
+      s3_bucket_arn          = module.datasync_opg_investigations_bucket.s3_bucket_arn
+    }
+  }
+
+  # schedule {
+  #   schedule_expression = "cron(0 23 ? * THU *)"
+  # }
+
+  tags = local.tags
+}

terraform/environments/analytical-platform-ingestion/iam-policies.tf

@@ -35,7 +35,7 @@ data "aws_iam_policy_document" "datasync" {
       "kms:DescribeKey",
       "kms:Decrypt",
     ]
-    resources = [module.transfer_logs_kms.key_arn]
+    resources = [module.s3_datasync_kms.key_arn]
   }
   statement {
     sid    = "AllowS3BucketActions"
@@ -45,7 +45,7 @@ data "aws_iam_policy_document" "datasync" {
       "s3:ListBucket",
       "s3:ListBucketMultipartUploads"
     ]
-    resources = [module.datasync_bucket.s3_bucket_arn]
+    resources = [module.datasync_opg_investigations_bucket.s3_bucket_arn]
   }
   statement {
     sid    = "AllowS3ObjectActions"
@@ -61,7 +61,7 @@ data "aws_iam_policy_document" "datasync" {
       "s3:PutObject",
       "s3:PutObjectTagging"
     ]
-    resources = ["${module.datasync_bucket.s3_bucket_arn}/*"]
+    resources = ["${module.datasync_opg_investigations_bucket.s3_bucket_arn}/*"]
   }
 }
 
@@ -116,7 +116,7 @@ data "aws_iam_policy_document" "datasync_replication" {
       "s3:GetReplicationConfiguration",
       "s3:ListBucket"
     ]
-    resources = [module.datasync_bucket.s3_bucket_arn]
+    resources = [module.datasync_opg_investigations_bucket.s3_bucket_arn]
   }
   statement {
     sid    = "SourceBucketObjectPermissions"
@@ -127,7 +127,7 @@ data "aws_iam_policy_document" "datasync_replication" {
       "s3:GetObjectVersionTagging",
       "s3:ObjectOwnerOverrideToBucketOwner"
     ]
-    resources = ["${module.datasync_bucket.s3_bucket_arn}/*"]
+    resources = ["${module.datasync_opg_investigations_bucket.s3_bucket_arn}/*"]
   }
 }
 

terraform/environments/analytical-platform-ingestion/s3.tf

@@ -161,57 +161,56 @@ module "bold_egress_bucket" {
   }
 }
 
-
-module "datasync_bucket" {
+module "datasync_opg_investigations_bucket" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
 
   source  = "terraform-aws-modules/s3-bucket/aws"
   version = "4.1.2"
 
-  bucket = "mojap-ingestion-${local.environment}-datasync"
+  bucket = "mojap-ingestion-${local.environment}-datasync-opg-investigations"
 
   force_destroy = true
 
-  versioning = {
-    enabled = true
-  }
-
-  replication_configuration = {
-    role = module.datasync_replication_iam_role.iam_role_arn
-    rules = [
-      {
-        id                        = "datasync-replication"
-        status                    = "Enabled"
-        delete_marker_replication = true
-
-        source_selection_criteria = {
-          sse_kms_encrypted_objects = {
-            enabled = true
-          }
-        }
-
-        destination = {
-          account_id    = local.environment_management.account_ids["analytical-platform-data-production"]
-          bucket        = "arn:aws:s3:::${local.environment_configuration.datasync_target_buckets[0]}"
-          storage_class = "STANDARD"
-          access_control_translation = {
-            owner = "Destination"
-          }
-          encryption_configuration = {
-            replica_kms_key_id = local.environment_configuration.mojap_land_kms_key
-          }
-          metrics = {
-            status  = "Enabled"
-            minutes = 15
-          }
-          replication_time = {
-            status  = "Enabled"
-            minutes = 15
-          }
-        }
-      }
-    ]
-  }
+  # versioning = {
+  #   enabled = true
+  # }
+
+  # replication_configuration = {
+  #   role = module.datasync_replication_iam_role.iam_role_arn
+  #   rules = [
+  #     {
+  #       id                        = "datasync-replication"
+  #       status                    = "Enabled"
+  #       delete_marker_replication = true
+
+  #       source_selection_criteria = {
+  #         sse_kms_encrypted_objects = {
+  #           enabled = true
+  #         }
+  #       }
+
+  #       destination = {
+  #         account_id    = local.environment_management.account_ids["analytical-platform-data-production"]
+  #         bucket        = "arn:aws:s3:::${local.environment_configuration.datasync_target_buckets[0]}"
+  #         storage_class = "STANDARD"
+  #         access_control_translation = {
+  #           owner = "Destination"
+  #         }
+  #         encryption_configuration = {
+  #           replica_kms_key_id = local.environment_configuration.mojap_land_kms_key
+  #         }
+  #         metrics = {
+  #           status  = "Enabled"
+  #           minutes = 15
+  #         }
+  #         replication_time = {
+  #           status  = "Enabled"
+  #           minutes = 15
+  #         }
+  #       }
+  #     }
+  #   ]
+  # }
 
   server_side_encryption_configuration = {
     rule = {

terraform/environments/analytical-platform-ingestion/security-groups.tf

@@ -90,7 +90,7 @@ module "datasync_activation_nlb_security_group" {
   vpc_id = module.connected_vpc.vpc_id
 
   egress_cidr_blocks = ["${local.environment_configuration.datasync_instance_private_ip}/32"]
-  egress_rules       = ["http-80-tcp", ]
+  egress_rules       = ["http-80-tcp"]
 
   ingress_cidr_blocks = ["${data.external.external_ip.result["ip"]}/32"]
   ingress_rules       = ["http-80-tcp"]
@@ -110,7 +110,7 @@ module "datasync_vpc_endpoint_security_group" {
   vpc_id = module.connected_vpc.vpc_id
 
   egress_cidr_blocks = [module.connected_vpc.vpc_cidr_block]
-  egress_rules       = ["all-all", ]
+  egress_rules       = ["all-all"]
 
   ingress_with_cidr_blocks = [
     {
@@ -160,6 +160,16 @@ module "datasync_instance_security_group" {
 
   vpc_id = module.connected_vpc.vpc_id
 
+  egress_with_cidr_blocks = [
+    {
+      from_port   = 445
+      to_port     = 445
+      protocol    = "tcp"
+      description = "SMB"
+      cidr_blocks = "10.0.0.0/8"
+    }
+  ]
+
   egress_with_source_security_group_id = [
     {
       from_port                = 1024

terraform/environments/digital-prison-reporting/application_variables.json

@@ -101,6 +101,8 @@
         "dps-inc-reporting",
         "dps-csip",
         "dps-alerts",
+        "dps-use-of-force",
+        "dps-locations",
         "dps-testing"
       ],
       "alarms": {
@@ -284,7 +286,9 @@
         "dps-basm",
         "dps-inc-reporting",
         "dps-csip",
-        "dps-alerts"
+        "dps-alerts",
+        "dps-use-of-force",
+        "dps-locations"
       ],
       "alarms": {
         "setup_cw_alarms": true,
@@ -469,7 +473,9 @@
         "dps-basm",
         "dps-inc-reporting",
         "dps-csip",
-        "dps-alerts"
+        "dps-alerts",
+        "dps-use-of-force",
+        "dps-locations"
       ],
       "alarms": {
         "setup_cw_alarms": true,
@@ -670,7 +676,9 @@
         "dps-basm",
         "dps-inc-reporting",
         "dps-csip",
-        "dps-alerts"
+        "dps-alerts",
+        "dps-use-of-force",
+        "dps-locations"
       ],
       "alarms": {
         "setup_cw_alarms": true,

terraform/environments/oasys/locals_production.tf

@@ -772,6 +772,8 @@ locals {
           { name = "t2-b", type = "CNAME", ttl = "3600", records = ["public-lb-1856376477.eu-west-2.elb.amazonaws.com"] },
           { name = "t2-b-int", type = "CNAME", ttl = "3600", records = ["internal-private-lb-1575012313.eu-west-2.elb.amazonaws.com"] },
           { name = "t2-int", type = "CNAME", ttl = "3600", records = ["internal-private-lb-1575012313.eu-west-2.elb.amazonaws.com"] },
+          { name = "t2-c", type = "CNAME", ttl = "3600", records = ["public-lb-1856376477.eu-west-2.elb.amazonaws.com"] },
+          { name = "t2-c-int", type = "CNAME", ttl = "3600", records = ["internal-private-lb-1575012313.eu-west-2.elb.amazonaws.com"] },
 
           { name = "_9f1b86e95d13d2cc7b9629f67d672c40", type = "CNAME", ttl = "86400", records = ["_7ea92a123c65795698dd19834dd71f61.fdbjvjdfdx.acm-validations.aws."] },
           { name = "_26aaae7b839510727c2dd323b483ea5d.pp", type = "CNAME", ttl = "86400", records = ["_72222d02a82256bb6d75c872bc7bc1aa.qxcwttcyyb.acm-validations.aws."] },