diff --git a/Project.toml b/Project.toml index a9d79b0..a994c8d 100644 --- a/Project.toml +++ b/Project.toml @@ -2,12 +2,10 @@ name = "AWSBatch" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" license = "MIT" authors = ["Invenia Technical Computing"] -version = "1.4.2" +version = "2.0.0" [deps] -AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" -AWSSDK = "0d499d91-6ae5-5d63-9313-12987b87d5ad" -AWSTools = "83bcdc74-1232-581c-948a-f29122bf8259" +AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" @@ -16,11 +14,8 @@ OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [compat] -AWSCore = "0.5.2, 0.6" -AWSSDK = "0.2, 0.3, 0.4, 0.5" -AWSTools = "0.3.1, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1" +AWS = "1" AutoHashEquals = "0.2.0" -HTTP = "0.8.1" Memento = "0.7, 0.8, 0.9, 0.10, 0.11, 0.12, 0.13, 1" Mocking = "0.7" OrderedCollections = "1.4" diff --git a/docs/src/index.md b/docs/src/index.md index 7967616..25fdd60 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -54,7 +54,7 @@ run_batch() ```@docs AWSBatch.BatchJob -AWSBatch.submit(::AbstractString, ::JobDefinition, ::AbstractString) +AWSBatch.submit AWSBatch.describe(::BatchJob) AWSBatch.JobDefinition(::BatchJob) AWSBatch.status(::BatchJob) @@ -67,6 +67,9 @@ AWSBatch.log_events(::BatchJob) ```@docs AWSBatch.JobDefinition +AWSBatch.ComputeEnvironment +AWSBatch.create_compute_environment +AWSBatch.list_job_definitions AWSBatch.job_definition_arn(::AbstractString) AWSBatch.register(::AbstractString) AWSBatch.deregister(::JobDefinition) @@ -74,6 +77,14 @@ AWSBatch.isregistered(::JobDefinition) AWSBatch.describe(::JobDefinition) ``` +### JobQueue + +```@docs +AWSBatch.JobQueue +AWSBatch.create_job_queue +AWSBatch.list_job_queues +``` + ### JobState ```@docs diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 9369f7f..08aac51 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -1,18 +1,19 @@ module AWSBatch +using AWS using AutoHashEquals -using AWSCore: AWSException, aws_config -using AWSSDK.Batch -using AWSSDK.CloudWatchLogs -using AWSTools.EC2: instance_region using OrderedCollections: OrderedDict using Dates using Memento using Mocking +@service Batch +@service Cloudwatch_Logs + export BatchJob, ComputeEnvironment, BatchEnvironmentError, BatchJobError -export JobQueue, JobDefinition, JobState +export JobQueue, JobDefinition, JobState, LogEvent export run_batch, describe, status, status_reason, wait, log_events, isregistered, register, deregister +export list_job_queues, list_job_definitions, create_compute_environment, create_job_queue const logger = getlogger(@__MODULE__) @@ -22,7 +23,6 @@ __init__() = Memento.register(logger) include("exceptions.jl") -include("version.jl") include("log_event.jl") include("compute_environment.jl") include("job_queue.jl") @@ -72,6 +72,7 @@ function run_batch(; num_jobs::Integer=1, parameters::Dict{String, String}=Dict{String, String}(), allow_job_registration::Bool=true, + aws_config::AbstractAWSConfig=global_aws_config(), ) if isa(definition, AbstractString) definition = isempty(definition) ? nothing : definition @@ -79,7 +80,7 @@ function run_batch(; # Determine if the job definition already exists and update the default job parameters if definition !== nothing - response = describe_job_definition(definition) + response = describe_job_definition(definition; aws_config=aws_config) if !isempty(response["jobDefinitions"]) details = first(response["jobDefinitions"]) @@ -101,11 +102,11 @@ function run_batch(; job_id = ENV["AWS_BATCH_JOB_ID"] job_queue = ENV["AWS_BATCH_JQ_NAME"] - # Get the zone information from the EC2 instance metadata. - isempty(region) && (region = @mock instance_region()) + # if not specified, get region from the aws_config + isempty(region) && (region = aws_config.region) # Requires permissions to access to "batch:DescribeJobs" - response = @mock describe_jobs(Dict("jobs" => [job_id])) + response = @mock Batch.describe_jobs([job_id]; aws_config=aws_config) # Use the job's description to only update fields that are using the default # values since explict arguments passed in via `kwargs` have higher priority @@ -165,8 +166,8 @@ function run_batch(; vcpus=vcpus, memory=memory, cmd=cmd, - region=region, parameters=parameters, + aws_config=aws_config, ) else throw(BatchEnvironmentError(string( @@ -193,10 +194,9 @@ function run_batch(; return submit( name, definition, - queue; + JobQueue(queue); container=container_overrides, parameters=parameters, - region=region, num_jobs=num_jobs, ) end diff --git a/src/batch_job.jl b/src/batch_job.jl index a0ae16c..bc97c79 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -1,6 +1,3 @@ -using AWSSDK.Batch: describe_jobs, submit_job -using AWSSDK.CloudWatchLogs: get_log_events - """ BatchJob @@ -23,10 +20,9 @@ end submit( name::AbstractString, definition::JobDefinition, - queue::AbstractString; + queue::JobQueue; container::AbstractDict=Dict(), parameters::Dict{String,String}=Dict{String, String}(), - region::AbstractString="", num_jobs::Integer=1, ) -> BatchJob @@ -35,23 +31,17 @@ Handles submitting the batch job. Returns a `BatchJob` wrapper for the id. function submit( name::AbstractString, definition::JobDefinition, - queue::AbstractString; + queue::JobQueue; container::AbstractDict=Dict(), parameters::Dict{String,String}=Dict{String, String}(), - region::AbstractString="", num_jobs::Integer=1, + aws_config::AbstractAWSConfig=global_aws_config(), ) - region = isempty(region) ? "us-east-1" : region - config = aws_config(region = region) - debug(logger, "Submitting job \"$name\"") - input = [ - "jobName" => name, - "jobQueue" => queue, - "jobDefinition" => definition.arn, + input = OrderedDict( "parameters" => parameters, "containerOverrides" => container, - ] + ) if num_jobs > 1 # https://docs.aws.amazon.com/batch/latest/userguide/array_jobs.html @@ -61,7 +51,7 @@ function submit( debug(logger, "Input: $input") - response = @mock submit_job(config, input) + response = @mock Batch.submit_job(definition.arn, name, queue.arn, input; aws_config=aws_config) job = BatchJob(response["jobId"]) if num_jobs > 1 @@ -78,8 +68,8 @@ end Provides details about the AWS batch job. """ -function describe(job::BatchJob) - response = @mock describe_jobs(; jobs=[job.id]) +function describe(job::BatchJob; aws_config::AbstractAWSConfig=global_aws_config()) + response = @mock Batch.describe_jobs([job.id]; aws_config=aws_config) isempty(response["jobs"]) && error(logger, "Job $(job.id) not found.") debug(logger, "Job $(job.id): $response") return first(response["jobs"]) @@ -90,15 +80,17 @@ end Returns the job definition corresponding to a batch job. """ -JobDefinition(job::BatchJob) = JobDefinition(describe(job)["jobDefinition"]) +function JobDefinition(job::BatchJob; aws_config::AbstractAWSConfig=global_aws_config()) + JobDefinition(describe(job)["jobDefinition"]; aws_config=aws_config) +end """ status(job::BatchJob) -> JobState Returns the current status of a job. """ -function status(job::BatchJob)::JobState - details = describe(job) +function status(job::BatchJob; aws_config::AbstractAWSConfig=global_aws_config())::JobState + details = describe(job; aws_config=aws_config) return parse(JobState, details["status"]) end @@ -108,8 +100,8 @@ end A short, human-readable string to provide additional details about the current status of the job. """ -function status_reason(job::BatchJob) - details = describe(job) +function status_reason(job::BatchJob; aws_config::AbstractAWSConfig=global_aws_config()) + details = describe(job; aws_config=aws_config) return get(details, "statusReason", nothing) end @@ -136,7 +128,8 @@ function Base.wait( cond::Function, job::BatchJob; timeout=600, - delay=5 + delay=5, + aws_config::AbstractAWSConfig=global_aws_config(), ) completed = false last_state = PENDING @@ -144,7 +137,7 @@ function Base.wait( start_time = time() # System time in seconds since epoch while time() - start_time < timeout - state = status(job) + state = status(job; aws_config=aws_config) if state != last_state || initial info(logger, "$(job.id) status $state") @@ -229,6 +222,3 @@ function log_events(job::BatchJob) return log_events("/aws/batch/job", stream) end -if @__VERSION__() < v"1.5" - @deprecate log_events(job::BatchJob, ::Union{Type{Vector{LogEvent}},Type{Nothing}}) log_events(job) -end diff --git a/src/compute_environment.jl b/src/compute_environment.jl index 27427fe..0aa118f 100644 --- a/src/compute_environment.jl +++ b/src/compute_environment.jl @@ -1,10 +1,16 @@ -using AWSSDK.Batch: describe_compute_environments +""" + ComputeEnvironment + +An object representing an AWS batch compute environment. + +See [`AWSBatch.create_compute_environment`](@ref). +""" struct ComputeEnvironment arn::String - function ComputeEnvironment(ce::AbstractString) - arn = compute_environment_arn(ce) + function ComputeEnvironment(ce::AbstractString; aws_config::AbstractAWSConfig=global_aws_config()) + arn = compute_environment_arn(ce; aws_config=aws_config) arn === nothing && error("No compute environment ARN found for $ce") new(arn) end @@ -12,19 +18,57 @@ end Base.:(==)(a::ComputeEnvironment, b::ComputeEnvironment) = a.arn == b.arn -describe(ce::ComputeEnvironment) = describe_compute_environment(ce) -max_vcpus(ce::ComputeEnvironment) = describe(ce)["computeResources"]["maxvCpus"] +function describe(ce::ComputeEnvironment; aws_config::AbstractAWSConfig=global_aws_config()) + describe_compute_environment(ce; aws_config=aws_config) +end +function max_vcpus(ce::ComputeEnvironment; aws_config::AbstractAWSConfig=global_aws_config()) + describe(ce; aws_config=aws_config)["computeResources"]["maxvCpus"] +end + +""" + create_compute_environment(name; + managed=true, + role="", + resources=Dict(), + enabled=true, + tags=Dict(), + aws_config=global_aws_config()) -function compute_environment_arn(ce::AbstractString) +Create a compute environment of type `type` with name `name`. + +See the AWS docs [here](https://docs.aws.amazon.com/batch/latest/APIReference/API_CreateComputeEnvironment.html). +""" +function create_compute_environment(name::AbstractString; + managed::Bool=true, + role::AbstractString="", + resources::AbstractDict=Dict{String,Any}(), + enabled::Bool=true, + tags::AbstractDict=Dict{String,Any}(), + aws_config::AbstractAWSConfig=global_aws_config()) + type = managed ? "MANAGED" : "UNMANAGED" + args = Dict{String,Any}() + isempty(role) || (args["serviceRole"] = role) + isempty(resources) || (args["computeResources"] = resources) + isempty(tags) || (args["tags"] = tags) + enabled || (args["state"] = "DISABLED") + return @mock Batch.create_compute_environment(name, type, args; aws_config=aws_config) +end + +function compute_environment_arn(ce::AbstractString; aws_config::AbstractAWSConfig=global_aws_config()) startswith(ce, "arn:") && return ce - json = describe_compute_environment(ce) + json = describe_compute_environment(ce; aws_config=aws_config) isempty(json) ? nothing : json["computeEnvironmentArn"] end -describe_compute_environment(ce::ComputeEnvironment) = describe_compute_environment(ce.arn) +function describe_compute_environment(ce::ComputeEnvironment; + aws_config::AbstractAWSConfig=global_aws_config()) + describe_compute_environment(ce.arn; aws_config=aws_config) +end -function describe_compute_environment(ce::AbstractString)::OrderedDict - json = @mock describe_compute_environments(Dict("computeEnvironments" => [ce])) +function describe_compute_environment(ce::AbstractString; + aws_config::AbstractAWSConfig=global_aws_config())::OrderedDict + json = @mock Batch.describe_compute_environments(Dict("computeEnvironments" => [ce]); + aws_config=aws_config) envs = json["computeEnvironments"] len = length(envs)::Int @assert len <= 1 diff --git a/src/job_definition.jl b/src/job_definition.jl index 616c29d..caeea77 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -1,6 +1,3 @@ -import AWSSDK.Batch: - describe_job_definitions, register_job_definition, deregister_job_definition - """ JobDefinition @@ -9,11 +6,11 @@ Stores the job definition arn including the revision. @auto_hash_equals struct JobDefinition arn::AbstractString - function JobDefinition(name::AbstractString) + function JobDefinition(name::AbstractString; aws_config::AbstractAWSConfig=global_aws_config()) if startswith(name, "arn:") new(name) else - arn = job_definition_arn(name) + arn = job_definition_arn(name; aws_config=aws_config) arn === nothing && error("No job definition ARN found for $name") new(arn) end @@ -24,7 +21,8 @@ end job_definition_arn( definition_name::AbstractString; image::AbstractString="", - role::AbstractString="" + role::AbstractString="", + aws_config::AbstractAWSConfig=global_aws_config(), ) -> Union{AbstractString, Nothing} Looks up the ARN (Amazon Resource Name) for the latest job definition that can be reused. @@ -41,8 +39,9 @@ function job_definition_arn( definition_name::AbstractString; image::AbstractString="", role::AbstractString="", + aws_config::AbstractAWSConfig=global_aws_config(), ) - response = describe_job_definition(definition_name) + response = describe_job_definition(definition_name; aws_config=aws_config) if !isempty(response["jobDefinitions"]) latest = first(response["jobDefinitions"]) @@ -97,31 +96,27 @@ function register( definition_name::AbstractString; image::AbstractString="", role::AbstractString="", + type::AbstractString="container", vcpus::Integer=1, memory::Integer=1024, cmd::Cmd=``, - region::AbstractString="", parameters::Dict{String, String}=Dict{String, String}(), + aws_config::AbstractAWSConfig=global_aws_config(), ) - region = isempty(region) ? "us-east-1" : region - config = aws_config(region = region) - debug(logger, "Registering job definition \"$definition_name\"") - input = [ - "type" => "container", + input = OrderedDict( "parameters" => parameters, - "containerProperties" => [ + "containerProperties" => OrderedDict( "image" => image, "vcpus" => vcpus, "memory" => memory, "command" => cmd.exec, "jobRoleArn" => role, - ], - "jobDefinitionName" => definition_name, - ] + ), + ) - response = @mock register_job_definition(config, input) - definition = JobDefinition(response["jobDefinitionArn"]) + response = @mock Batch.register_job_definition(definition_name, type, input; aws_config=aws_config) + definition = JobDefinition(response["jobDefinitionArn"]; aws_config=aws_config) info(logger, "Registered job definition \"$(definition.arn)\"") return definition end @@ -131,36 +126,53 @@ end Deregisters an AWS Batch job. """ -function deregister(definition::JobDefinition) +function deregister(definition::JobDefinition; aws_config::AbstractAWSConfig=global_aws_config()) debug(logger, "Deregistering job definition \"$(definition.arn)\"") - resp = deregister_job_definition(Dict("jobDefinition" => definition.arn)) + resp = @mock Batch.deregister_job_definition(definition.arn; aws_config=aws_config) info(logger, "Deregistered job definition \"$(definition.arn)\"") end """ - isregistered(definition::JobDefinition) -> Bool + isregistered(definition::JobDefinition; aws_config=global_aws_config()) -> Bool Checks if a JobDefinition is registered. """ -function isregistered(definition::JobDefinition) - j = describe(definition) +function isregistered(definition::JobDefinition; aws_config::AbstractAWSConfig=global_aws_config()) + j = describe(definition; aws_config=aws_config) return any(d -> d["status"] == "ACTIVE", get(j, "jobDefinitions", [])) end """ - describe(definition::JobDefinition) -> Dict + list_job_definitions(;aws_config=global_aws_config()) + +Get a list of `JobDefinition` objects via `Batch.decsribe_job_definitions()`. +""" +function list_job_definitions(;aws_config::AbstractAWSConfig=global_aws_config()) + job_definitions = Batch.describe_job_definitions(; aws_config=aws_config)["jobDefinitions"] + + return [JobDefintiion(jd["jobDefinitionArn"]) for jd in job_definitions] +end + +""" + describe(definition::JobDefinition; aws_config=global_aws_config()) -> Dict Describes a job definition as a dictionary. Requires the IAM permissions "batch:DescribeJobDefinitions". """ -describe(definition::JobDefinition) = describe_job_definition(definition) +function describe(definition::JobDefinition; aws_config::AbstractAWSConfig=global_aws_config()) + describe_job_definition(definition; aws_config=aws_config) +end -describe_job_definition(definition::JobDefinition) = describe_job_definition(definition.arn) -function describe_job_definition(definition::AbstractString) +function describe_job_definition(definition::JobDefinition; + aws_config::AbstractAWSConfig=global_aws_config()) + describe_job_definition(definition.arn; aws_config=aws_config) +end +function describe_job_definition(definition::AbstractString; + aws_config::AbstractAWSConfig=global_aws_config()) query = if startswith(definition, "arn:") Dict("jobDefinitions" => [definition]) else Dict("jobDefinitionName" => definition) end - return @mock describe_job_definitions(query) + return @mock Batch.describe_job_definitions(query; aws_config=aws_config) end diff --git a/src/job_queue.jl b/src/job_queue.jl index dc6e8bf..5e65a27 100644 --- a/src/job_queue.jl +++ b/src/job_queue.jl @@ -1,23 +1,75 @@ -using AWSSDK.Batch: describe_job_queues +""" + JobQueue + +An object representing and AWS batch job queue. + +See [`AWSBatch.create_job_queue`](@ref). +""" struct JobQueue arn::String - function JobQueue(queue::AbstractString) - arn = job_queue_arn(queue) + function JobQueue(queue::AbstractString; aws_config::AbstractAWSConfig=global_aws_config()) + arn = job_queue_arn(queue; aws_config=aws_config) arn === nothing && error("No job queue ARN found for: $queue") new(arn) end end Base.:(==)(a::JobQueue, b::JobQueue) = a.arn == b.arn -describe(queue::JobQueue) = describe_job_queue(queue) -describe_job_queue(queue::JobQueue) = describe_job_queue(queue.arn) -max_vcpus(queue::JobQueue) = sum(max_vcpus(ce) for ce in compute_environments(queue)) +function describe(queue::JobQueue; aws_config::AbstractAWSConfig=global_aws_config()) + return describe_job_queue(queue; aws_config=aws_config) +end +function describe_job_queue(queue::JobQueue; aws_config::AbstractAWSConfig=global_aws_config()) + return describe_job_queue(queue.arn; aws_config=aws_config) +end +function max_vcpus(queue::JobQueue; aws_config::AbstractAWSConfig=global_aws_config()) + sum(max_vcpus(ce; aws_config=aws_config) for ce in compute_environments(queue; aws_config=aws_config)) +end + +function _create_compute_environment_order(envs) + map(enumerate(envs)) do (i, env) + Dict{String,Any}("computeEnvironment"=>env, "order"=>i) + end +end + +""" + create_job_queue(name, envs, priority=1; aws_config=global_aws_config()) + +Create a job queue with name `name` and priority `priority` returning the associated `JobQueue` object. +`envs` must be an iterator of compute environments given by ARN. + +See the AWS docs [here](https://docs.aws.amazon.com/batch/latest/APIReference/API_CreateJobQueue.html). +""" +function create_job_queue(name::AbstractString, envs, priority::Integer=1; + enabled::Bool=true, + tags::AbstractDict=Dict{String,Any}(), + aws_config::AbstractAWSConfig=global_aws_config()) + env = _create_compute_environment_order(envs) + args = Dict{String,Any}() + enabled || (args["state"] = "DISABLED") + isempty(tags) || (args["tags"] = tags) + return @mock Batch.create_job_queue(env, name, priority, args; aws_config=aws_config) +end + + +""" + list_job_queues(;aws_config=global_aws_config()) + +Get a list of `JobQueue` objects as returned by `Batch.describe_job_queues()`. +""" +function list_job_queues(;aws_config::AbstractAWSConfig=global_aws_config()) + [JobQueue(q["jobQueueArn"]) for q ∈ Batch.describe_job_queues(;aws_config=aws_config)["jobQueues"]] +end + +""" + compute_environments(queue::JobQueue; aws_config=global_aws_config()) -function compute_environments(queue::JobQueue) - ce_order = describe(queue)["computeEnvironmentOrder"] +Get a list of `ComputeEnvironment` objects associated with the `JobQueue`. +""" +function compute_environments(queue::JobQueue; aws_config::AbstractAWSConfig=global_aws_config()) + ce_order = describe(queue; aws_config=aws_config)["computeEnvironmentOrder"] compute_envs = Vector{ComputeEnvironment}(undef, length(ce_order)) for ce in ce_order @@ -29,15 +81,16 @@ function compute_environments(queue::JobQueue) end -function job_queue_arn(queue::AbstractString) +function job_queue_arn(queue::AbstractString; aws_config::AbstractAWSConfig=global_aws_config()) startswith(queue, "arn:") && return queue - json = describe_job_queue(queue) + json = describe_job_queue(queue; aws_config=aws_config) isempty(json) ? nothing : json["jobQueueArn"] end -function describe_job_queue(queue::AbstractString)::OrderedDict - json = @mock describe_job_queues(Dict("jobQueues" => [queue])) +function describe_job_queue(queue::AbstractString; + aws_config::AbstractAWSConfig=global_aws_config())::OrderedDict + json = @mock Batch.describe_job_queues(Dict("jobQueues" => [queue]); aws_config=aws_config) queues = json["jobQueues"] len = length(queues)::Int @assert len <= 1 diff --git a/src/log_event.jl b/src/log_event.jl index cba99cf..5a4019d 100644 --- a/src/log_event.jl +++ b/src/log_event.jl @@ -36,7 +36,8 @@ end Fetches the CloudWatch log from the specified log group and stream as a `Vector` of `LogEvent`s. If the log stream does not exist then `nothing` will be returned. """ -function log_events(log_group::AbstractString, log_stream::AbstractString) +function log_events(log_group::AbstractString, log_stream::AbstractString; + aws_config::AbstractAWSConfig=global_aws_config()) events = LogEvent[] curr_token = nothing @@ -45,17 +46,17 @@ function log_events(log_group::AbstractString, log_stream::AbstractString) # We've hit the end of the stream if the next token matches the current one. while next_token != curr_token || next_token === nothing response = try - @mock get_log_events( - logGroupName=log_group, - logStreamName=log_stream, - nextToken=next_token, + @mock Cloudwatch_Logs.get_log_events( + log_group, log_stream, + Dict("nextToken"=>next_token); + aws_config=aws_config, ) catch e # The specified log stream does not exist. Specifically, this can occur when # a batch job has a reference to a log stream but the stream has not yet been # created. if ( - e isa AWSException && + e isa AWSExceptions.AWSException && e.cause.status == 400 && e.info["message"] == "The specified log stream does not exist." ) diff --git a/src/version.jl b/src/version.jl deleted file mode 100644 index ee69479..0000000 --- a/src/version.jl +++ /dev/null @@ -1,29 +0,0 @@ -# TODO: Temporary until this is turned into a package -using Pkg: Pkg - -# https://github.com/JuliaLang/julia/pull/33128 -if VERSION < v"1.4.0-DEV.397" - function pkgdir(m::Module) - rootmodule = Base.moduleroot(m) - path = pathof(rootmodule) - path === nothing && return nothing - return dirname(dirname(path)) - end -end - -""" - @__VERSION__ -> Union{VersionNumber, Nothing} - -Get the `VersionNumber` of the package which expands this macro. If executed outside of a -package `nothing` will be returned. -""" -macro __VERSION__() - pkg_dir = pkgdir(__module__) - - if pkg_dir !== nothing - project_data = Pkg.TOML.parsefile(Pkg.Types.projectfile_path(pkg_dir)) - return VersionNumber(project_data["version"]) - else - return nothing - end -end diff --git a/test/batch_job.jl b/test/batch_job.jl index d29da52..c7fdd1e 100644 --- a/test/batch_job.jl +++ b/test/batch_job.jl @@ -3,7 +3,7 @@ @testset "status_reason" begin @testset "not provided" begin - patch = @patch function describe_jobs(; kwargs...) + patch = @patch function AWSBatch.Batch.describe_jobs(args...; kwargs...) Dict( "jobs" => [ Dict() @@ -18,7 +18,7 @@ @testset "provided" begin reason = "Essential container in task exited" - patch = @patch function describe_jobs(; kwargs...) + patch = @patch function AWSBatch.Batch.describe_jobs(args...; kwargs...) Dict( "jobs" => [ Dict("statusReason" => reason) @@ -48,7 +48,7 @@ function status_patch(states) index = 1 - return @patch function describe_jobs(; kwargs...) + return @patch function AWSBatch.Batch.describe_jobs(args...; kwargs...) json = Dict( "jobs" => [ Dict("status" => states[index]) diff --git a/test/compute_environment.jl b/test/compute_environment.jl index 3cf354b..20ed13c 100644 --- a/test/compute_environment.jl +++ b/test/compute_environment.jl @@ -3,14 +3,17 @@ using OrderedCollections: OrderedDict @testset "ComputeEnvironment" begin @testset "constructor" begin arn = "arn:aws:batch:us-east-1:000000000000:compute-environment/ce" - @test ComputeEnvironment(arn).arn == arn - patch = describe_compute_environments_patch( OrderedDict( "computeEnvironmentName" => "ce-name", "computeEnvironmentArn" => arn, ) ) + + apply(patch) do + @test ComputeEnvironment(arn).arn == arn + end + apply(patch) do @test ComputeEnvironment("ce-name").arn == arn end diff --git a/test/job_queue.jl b/test/job_queue.jl index c64472c..5e1593f 100644 --- a/test/job_queue.jl +++ b/test/job_queue.jl @@ -3,14 +3,17 @@ using OrderedCollections: OrderedDict @testset "JobQueue" begin @testset "constructor" begin arn = "arn:aws:batch:us-east-1:000000000000:job-queue/queue" - @test JobQueue(arn).arn == arn - patch = describe_job_queues_patch( OrderedDict( "jobQueueName" => "queue-name", "jobQueueArn" => arn, ) ) + + apply(patch) do + @test JobQueue(arn).arn == arn + end + apply(patch) do @test JobQueue("queue-name").arn == arn end diff --git a/test/mock.jl b/test/mock.jl index d7a06ed..ffd3f73 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -97,7 +97,7 @@ const DESCRIBE_JOBS_RESP = Dict( ) function describe_compute_environments_patch(output::Vector=[]) - @patch function AWSBatch.describe_compute_environments(d::Dict) + @patch function AWSBatch.Batch.describe_compute_environments(d::Dict; aws_config=aws_config) compute_envs = d["computeEnvironments"] @assert length(compute_envs) == 1 ce = first(compute_envs) @@ -113,7 +113,7 @@ function describe_compute_environments_patch(output::OrderedDict) end function describe_job_queues_patch(output::Vector=[]) - @patch function AWSBatch.describe_job_queues(d::Dict) + @patch function AWSBatch.Batch.describe_job_queues(d::Dict; aws_config=aws_config) queues = d["jobQueues"] @assert length(queues) == 1 queue = first(queues) @@ -136,10 +136,10 @@ function log_events_patches(; log_stream_name="mock_stream", events=[], exceptio end get_log_events_patch = if exception !== nothing - @patch get_log_events(; kwargs...) = throw(exception) + @patch AWSBatch.Cloudwatch_Logs.get_log_events(args...; kwargs...) = throw(exception) else - @patch function get_log_events(; kwargs...) - if get(kwargs, :nextToken, nothing) === nothing + @patch function AWSBatch.Cloudwatch_Logs.get_log_events(grp, stream, params; kwargs...) + if get(params, "nextToken", nothing) === nothing Dict("events" => events, "nextForwardToken" => "0") else Dict("events" => [], "nextForwardToken" => "0") @@ -148,7 +148,7 @@ function log_events_patches(; log_stream_name="mock_stream", events=[], exceptio end return [ - @patch describe_jobs(args...; kwargs...) = job_descriptions + @patch AWSBatch.Batch.describe_jobs(args...; kwargs...) = job_descriptions get_log_events_patch ] end diff --git a/test/run_batch.jl b/test/run_batch.jl index a0eaf3d..a49cb9d 100644 --- a/test/run_batch.jl +++ b/test/run_batch.jl @@ -1,10 +1,17 @@ -function _register_job_def(config::AWSConfig, input::AbstractArray, expected::AbstractArray) - @test input == expected +function _register_job_def(name, type, input, expected) + @test name == expected["jobDefinitionName"] + @test type == expected["type"] + @test input["parameters"] == expected["parameters"] + @test input["containerProperties"] == expected["containerProperties"] return REGISTER_JOB_DEF_RESP end -function _submit_job(config::AWSConfig, input::AbstractArray, expected::AbstractArray) - @test input == expected +function _submit_job(def, name, queue, input, expected=AbstractDict) + @test def == expected["jobDefinition"] + @test name == expected["jobName"] + @test queue == expected["jobQueue"] + @test input["parameters"] == expected["parameters"] + @test input["containerOverrides"] == expected["containerOverrides"] return SUBMIT_JOB_RESP end @@ -17,10 +24,16 @@ end end end + queue_arn = "arn:aws:batch:us-east-1:000000000000:job-queue/HighPriority" + queue_patch = describe_job_queues_patch(OrderedDict("jobQueueName"=>"HighPriority", + "jobQueueArn"=>queue_arn)) + + aws_config = global_aws_config() + @testset "From Job Definition" begin - expected_job = [ + expected_job = OrderedDict( "jobName" => "example", - "jobQueue" => "HighPriority", + "jobQueue" => queue_arn, "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", "parameters" => Dict{String,String}(), "containerOverrides" => Dict( @@ -28,11 +41,13 @@ end "memory" => 128, "vcpus" => 1, ), - ] + ) patches = [ - @patch describe_job_definitions(args...) = DESCRIBE_JOBS_DEF_RESP - @patch submit_job(config, input) = _submit_job(config, input, expected_job) + queue_patch + @patch AWSBatch.Batch.describe_job_definitions(args...; kw...) = DESCRIBE_JOBS_DEF_RESP + @patch AWSBatch.Batch.submit_job(def, name, queue, input; kw...) = + _submit_job(def, name, queue, input, expected_job) ] apply(patches) do @@ -43,9 +58,9 @@ end @testset "From Current Job" begin withenv(BATCH_ENVS...) do - expected_job = [ + expected_job = OrderedDict( "jobName" => "example", - "jobQueue" => "HighPriority", + "jobQueue" => queue_arn, "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", "parameters" => Dict{String,String}(), "containerOverrides" => Dict( @@ -53,28 +68,29 @@ end "memory" => 128, "vcpus" => 1, ), - ] + ) - expected_job_def = [ + expected_job_def = OrderedDict( "type" => "container", "parameters" => Dict{String,String}(), - "containerProperties" => [ + "containerProperties" => OrderedDict( "image" => "busybox", "vcpus" => 1, "memory" => 128, "command" => ["sleep", "60"], "jobRoleArn" => "arn:aws:iam::012345678910:role/sleep60", - ], + ), "jobDefinitionName" => "sleep60", - ] + ) patches = [ - @patch instance_region() = "us-east-1" - @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP - @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) - @patch register_job_definition(config, input) = - _register_job_def(config, input, expected_job_def) - @patch submit_job(config, input) = _submit_job(config, input, expected_job) + queue_patch + @patch AWSBatch.Batch.describe_jobs(args...; kw...) = DESCRIBE_JOBS_RESP + @patch AWSBatch.Batch.describe_job_definitions(args...; kw...) = Dict("jobDefinitions" => Dict()) + @patch AWSBatch.Batch.register_job_definition(name, type, input; kw...) = + _register_job_def(name, type, input, expected_job_def) + @patch AWSBatch.Batch.submit_job(def, name, queue, input; kw...) = + _submit_job(def, name, queue, input, expected_job) ] apply(patches) do @@ -86,9 +102,9 @@ end @testset "Using a Job Definition" begin withenv(BATCH_ENVS...) do - expected_job = [ + expected_job = OrderedDict( "jobName" => "example", - "jobQueue" => "HighPriority", + "jobQueue" => queue_arn, "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", "parameters" => Dict{String,String}(), "containerOverrides" => Dict( @@ -96,13 +112,14 @@ end "memory" => 128, "vcpus" => 1, ), - ] + ) patches = [ - @patch instance_region() = "us-east-1" - @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP - @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) - @patch submit_job(config, input) = _submit_job(config, input, expected_job) + queue_patch + @patch AWSBatch.Batch.describe_jobs(args...; kw...) = DESCRIBE_JOBS_RESP + @patch AWSBatch.Batch.describe_job_definitions(args...; kw...) = Dict("jobDefinitions" => Dict()) + @patch AWSBatch.Batch.submit_job(def, name, queue, input; kw...) = + _submit_job(def, name, queue, input, expected_job) ] apply(patches) do diff --git a/test/runtests.jl b/test/runtests.jl index d1c84e7..0b74ece 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,10 +1,5 @@ +using AWS using AWSBatch -using AWSBatch: LogEvent, describe_jobs, describe_job_definitions, register_job_definition, - submit_job -using AWSCore: AWSConfig, AWSException -using AWSSDK.CloudWatchLogs: get_log_events -using AWSTools.CloudFormation: stack_output -using AWSTools.EC2: instance_region using Dates using HTTP: HTTP using Memento @@ -12,8 +7,14 @@ using Memento.TestUtils: @test_log, @test_nolog using Mocking using Test +using AWS.AWSExceptions: AWSException + Mocking.activate() +# need to define these to make sure we don't inadvertently try to talk to AWS +ENV["AWS_ACCESS_KEY_ID"] = "" +ENV["AWS_SECRET_ACCESS_KEY"] = "" + # Controls the running of various tests: "local", "batch" const TESTS = strip.(split(get(ENV, "TESTS", "local"), r"\s*,\s*")) @@ -192,7 +193,6 @@ include("mock.jl") vcpus=1, memory=1024, cmd=command, - region="us-east-1", parameters=Dict("juliacmd" => "println(\"Default String\")"), )