rocm_bandwidth_test_trans.cpp

////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
//                 AMD Research and AMD HSA Software Development
//
//                 Advanced Micro Devices, Inc.
//
//                 www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
//  - Redistributions of source code must retain the above copyright notice,
//    this list of conditions and the following disclaimers.
//  - Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimers in
//    the documentation and/or other materials provided with the distribution.
//  - Neither the names of Advanced Micro Devices, Inc,
//    nor the names of its contributors may be used to endorse or promote
//    products derived from this Software without specific prior written
//    permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////

#include "common.hpp"
#include "rocm_bandwidth_test.hpp"

bool RocmBandwidthTest::FindMirrorRequest(bool reverse, uint32_t src_idx, uint32_t dst_idx) {
    uint32_t size = trans_list_.size();
    for (uint32_t idx = 0; idx < size; idx++) {
        async_trans_t& mirror = trans_list_[idx];
        if (reverse) {
            if ((src_idx == mirror.copy.dst_idx_) && (dst_idx == mirror.copy.src_idx_)) {
                return true;
            }
        } else {
            if ((src_idx == mirror.copy.src_idx_) && (dst_idx == mirror.copy.dst_idx_)) {
                return true;
            }
        }
    }

    return false;
}

bool RocmBandwidthTest::BuildReadOrWriteTrans(uint32_t req_type, vector<size_t>& in_list) {
    // Validate the list of pool-agent tuples
    hsa_status_t status;
    hsa_amd_memory_pool_access_t access;
    uint32_t list_size = in_list.size();
    for (uint32_t idx = 0; idx < list_size; idx += 2) {
        uint32_t pool_idx = in_list[idx];
        uint32_t exec_idx = in_list[idx + 1];

        // Retrieve Roc runtime handles for memory pool and agent
        hsa_agent_t exec_agent = agent_list_[exec_idx].agent_;
        hsa_amd_memory_pool_t pool = pool_list_[pool_idx].pool_;

        // Determine agent can access the memory pool
        status = hsa_amd_agent_memory_pool_get_info(exec_agent, pool,
                                                    HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS, &access);
        ErrorCheck(status);

        // Determine if accessibility to agent is not denied
        if (access == HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED) {
            PrintIOAccessError(exec_idx, pool_idx);
            return false;
        }

        // Agent has access, build an instance of transaction
        // and add it to the list of transactions
        async_trans_t trans(req_type);
        trans.kernel.code_ = NULL;
        trans.kernel.pool_ = pool;
        trans.kernel.pool_idx_ = pool_idx;
        trans.kernel.agent_ = exec_agent;
        trans.kernel.agent_idx_ = exec_idx;
        trans_list_.push_back(trans);
    }
    return true;
}

bool RocmBandwidthTest::BuildReadTrans() { return BuildReadOrWriteTrans(REQ_READ, read_list_); }

bool RocmBandwidthTest::BuildWriteTrans() { return BuildReadOrWriteTrans(REQ_WRITE, write_list_); }

bool RocmBandwidthTest::FilterCpuPool(uint32_t req_type, hsa_device_type_t dev_type,
                                      bool fine_grained) {
    if ((req_type != REQ_COPY_ALL_BIDIR) && (req_type != REQ_COPY_ALL_UNIDIR)) {
        return false;
    }

    // Determine if device is a Cpu - filter out only if
    // it is a Cpu device
    if (dev_type != HSA_DEVICE_TYPE_CPU) {
        return false;
    }

    // If env to skip fine grain is NULL it means
    // we should filter out coarse-grain pools
    if (skip_cpu_fine_grain_ == NULL) {
        return (fine_grained == false);
    }

    // If env to skip fine grain is NON-NULL it means
    // we should filter out fine-grain pools
    return (fine_grained == true);
}

bool RocmBandwidthTest::BuildCopyTrans(uint32_t req_type, vector<size_t>& src_list,
                                       vector<size_t>& dst_list) {
    uint32_t src_size = src_list.size();
    uint32_t dst_size = dst_list.size();

    for (uint32_t idx = 0; idx < src_size; idx++) {
        // Retrieve Roc runtime handles for Src memory pool and agents
        uint32_t src_idx = src_list[idx];
        uint32_t src_dev_idx = pool_list_[src_idx].agent_index_;
        hsa_amd_memory_pool_t src_pool = pool_list_[src_idx].pool_;
        hsa_device_type_t src_dev_type = agent_list_[src_dev_idx].device_type_;

        for (uint32_t jdx = 0; jdx < dst_size; jdx++) {
            // Retrieve Roc runtime handles for Dst memory pool and agents
            uint32_t dst_idx = dst_list[jdx];
            uint32_t dst_dev_idx = pool_list_[dst_idx].agent_index_;
            hsa_amd_memory_pool_t dst_pool = pool_list_[dst_idx].pool_;
            hsa_device_type_t dst_dev_type = agent_list_[dst_dev_idx].device_type_;

            // Filter out transactions that involve only Cpu agents/devices
            // without regard to type of request, default run, partial or full
            // unidirectional or bidirectional copies
            if ((src_dev_type == HSA_DEVICE_TYPE_CPU) && (dst_dev_type == HSA_DEVICE_TYPE_CPU)) {
                continue;
            }

            // Filter out transactions that involve only same GPU as both
            // Src and Dst device if the request is bidirectional copy that
            // is either partial or full
            if ((req_type == REQ_COPY_BIDIR) || (req_type == REQ_COPY_ALL_BIDIR)) {
                if (src_dev_idx == dst_dev_idx) {
                    continue;
                }

                bool mirror = FindMirrorRequest(true, src_idx, dst_idx);
                if (mirror) {
                    continue;
                }
            }

            // Determine if accessibility to dst pool for src agent is not denied
            uint32_t path_exists = access_matrix_[(src_dev_idx * agent_index_) + dst_dev_idx];
            if (path_exists == 0) {
                if ((req_type == REQ_COPY_ALL_BIDIR) || (req_type == REQ_COPY_ALL_UNIDIR)) {
                    continue;
                } else {
                    PrintCopyAccessError(src_idx, dst_idx);
                    return false;
                }
            }

            // For bidirectional copies determine both access paths are valid
            // Both paths are valid when one of the devices is a CPU. This is
            // not true when both of the devices are GPU's.
            if ((req_type == REQ_COPY_ALL_BIDIR) || (req_type == REQ_COPY_ALL_UNIDIR)) {
                path_exists = access_matrix_[(dst_dev_idx * agent_index_) + src_dev_idx];
                if (path_exists == 0) {
                    continue;
                }
            }

            // Update the list of agents active in any copy operation
            if (active_agents_list_ == NULL) {
                active_agents_list_ = new uint32_t[agent_index_]();
            }
            active_agents_list_[src_dev_idx] = 1;
            active_agents_list_[dst_dev_idx] = 1;

            // Agents have access, build an instance of transaction
            // and add it to the list of transactions
            async_trans_t trans(req_type);
            trans.copy.src_idx_ = src_idx;
            trans.copy.dst_idx_ = dst_idx;
            trans.copy.src_pool_ = src_pool;
            trans.copy.dst_pool_ = dst_pool;
            trans.copy.bidir_ = ((req_type == REQ_COPY_BIDIR) || (req_type == REQ_COPY_ALL_BIDIR));
            trans.copy.uses_gpu_ =
                ((src_dev_type == HSA_DEVICE_TYPE_GPU) || (dst_dev_type == HSA_DEVICE_TYPE_GPU));
            trans_list_.push_back(trans);
        }
    }

    return true;
}

bool RocmBandwidthTest::BuildConcurrentCopyTrans(uint32_t req_type, vector<size_t>& dev_list) {
    uint32_t size = dev_list.size();
    for (uint32_t idx = 0; idx < size; idx += 2) {
        // Retrieve Roc runtime handles for Src memory pool and agents
        uint32_t src_idx = dev_list[idx];
        uint32_t src_dev_idx = pool_list_[src_idx].agent_index_;
        hsa_amd_memory_pool_t src_pool = pool_list_[src_idx].pool_;
        hsa_device_type_t src_dev_type = agent_list_[src_dev_idx].device_type_;

        // Retrieve Roc runtime handles for Dst memory pool and agents
        uint32_t dst_idx = dev_list[idx + 1];
        uint32_t dst_dev_idx = pool_list_[dst_idx].agent_index_;
        hsa_amd_memory_pool_t dst_pool = pool_list_[dst_idx].pool_;
        hsa_device_type_t dst_dev_type = agent_list_[dst_dev_idx].device_type_;

        // Filter out transactions that involve only Cpu agents/devices
        // without regard to type of request, default run, partial or full
        // unidirectional or bidirectional copies
        if ((src_dev_type == HSA_DEVICE_TYPE_CPU) && (dst_dev_type == HSA_DEVICE_TYPE_CPU)) {
            continue;
        }

        // Determine there is no duplicate
        bool mirror = false;
        mirror = FindMirrorRequest(false, src_idx, dst_idx);
        if (mirror) {
            continue;
        }

        // Filter out transactions that involve only same GPU as both
        // Src and Dst device if the request is bidirectional copy that
        // is either partial or full
        if (req_type == REQ_CONCURRENT_COPY_BIDIR) {
            if (src_dev_idx == dst_dev_idx) {
                continue;
            }

            mirror = FindMirrorRequest(true, src_idx, dst_idx);
            if (mirror) {
                continue;
            }
        }

        // Determine if accessibility to dst pool for src agent is not denied
        uint32_t path_exists = access_matrix_[(src_dev_idx * agent_index_) + dst_dev_idx];
        if (path_exists == 0) {
            PrintCopyAccessError(src_idx, dst_idx);
            return false;
        }

        // For bidirectional copies determine both access paths are valid
        // Both paths are valid when one of the devices is a CPU. This is
        // not true when both of the devices are GPU's.
        if (req_type == REQ_CONCURRENT_COPY_BIDIR) {
            path_exists = access_matrix_[(dst_dev_idx * agent_index_) + src_dev_idx];
            if (path_exists == 0) {
                PrintCopyAccessError(dst_idx, src_idx);
                return false;
            }
        }

        // Update the list of agents active in any copy operation
        if (active_agents_list_ == NULL) {
            active_agents_list_ = new uint32_t[agent_index_]();
        }
        active_agents_list_[src_dev_idx] = 1;
        active_agents_list_[dst_dev_idx] = 1;

        // Agents have access, build an instance of transaction
        // and add it to the list of transactions
        async_trans_t trans(req_type);
        trans.copy.src_idx_ = src_idx;
        trans.copy.dst_idx_ = dst_idx;
        trans.copy.src_pool_ = src_pool;
        trans.copy.dst_pool_ = dst_pool;
        trans.copy.bidir_ = (req_type == REQ_CONCURRENT_COPY_BIDIR);
        trans.copy.uses_gpu_ =
            ((src_dev_type == HSA_DEVICE_TYPE_GPU) || (dst_dev_type == HSA_DEVICE_TYPE_GPU));
        trans_list_.push_back(trans);
    }

    return true;
}

bool RocmBandwidthTest::BuildBidirCopyTrans() {
    return BuildCopyTrans(REQ_COPY_BIDIR, bidir_list_, bidir_list_);
}

bool RocmBandwidthTest::BuildUnidirCopyTrans() {
    return BuildCopyTrans(REQ_COPY_UNIDIR, src_list_, dst_list_);
}

bool RocmBandwidthTest::BuildAllPoolsBidirCopyTrans() {
    return BuildCopyTrans(REQ_COPY_ALL_BIDIR, bidir_list_, bidir_list_);
}

bool RocmBandwidthTest::BuildAllPoolsUnidirCopyTrans() {
    return BuildCopyTrans(REQ_COPY_ALL_UNIDIR, src_list_, dst_list_);
}

// @brief: Builds a list of transaction per user request
bool RocmBandwidthTest::BuildTransList() {
    // Build list of Read transactions per user request
    if (req_read_ == REQ_READ) {
        return BuildReadTrans();
    }

    // Build list of Write transactions per user request
    if (req_write_ == REQ_WRITE) {
        return BuildWriteTrans();
    }

    // Build list of Bidirectional Copy transactions per user request
    if (req_copy_bidir_ == REQ_COPY_BIDIR) {
        return BuildBidirCopyTrans();
    }

    // Build list of Unidirectional Copy transactions per user request
    if (req_copy_unidir_ == REQ_COPY_UNIDIR) {
        return BuildUnidirCopyTrans();
    }

    // Build list of All Bidir Copy transactions per user request
    if (req_copy_all_bidir_ == REQ_COPY_ALL_BIDIR) {
        return BuildAllPoolsBidirCopyTrans();
    }

    // Build list of All Unidir Copy transactions per user request
    if (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR) {
        return BuildAllPoolsUnidirCopyTrans();
    }

    // Build list of Bidir Concurrent Copy transactions per user request
    if (req_concurrent_copy_bidir_ == REQ_CONCURRENT_COPY_BIDIR) {
        return BuildConcurrentCopyTrans(req_concurrent_copy_bidir_, bidir_list_);
    }

    // Build list of Unidir Concurrent Copy transactions per user request
    if (req_concurrent_copy_unidir_ == REQ_CONCURRENT_COPY_UNIDIR) {
        return BuildConcurrentCopyTrans(req_concurrent_copy_unidir_, bidir_list_);
    }

    // All of the transaction are built up
    return true;
}

void RocmBandwidthTest::ComputeCopyTime(std::vector<async_trans_t>& trans_list) {
    uint32_t trans_cnt = trans_list.size();
    for (uint32_t idx = 0; idx < trans_cnt; idx++) {
        async_trans_t& trans = trans_list[idx];
        ComputeCopyTime(trans);
    }
}

void RocmBandwidthTest::ComputeCopyTime(async_trans_t& trans) {
    // Get the frequency of Gpu Timestamping
    uint64_t sys_freq = 0;
    hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &sys_freq);

    double avg_time = 0;
    double min_time = 0;
    size_t data_size = 0;
    double avg_bandwidth = 0;
    double peak_bandwidth = 0;
    uint32_t size_len = size_list_.size();
    for (uint32_t idx = 0; idx < size_len; idx++) {
        // Adjust size of data involved in copy
        data_size = size_list_[idx];
        if (trans.copy.bidir_ == true) {
            data_size += size_list_[idx];
        }

        // Double data size if copying the same device
        if (trans.copy.src_idx_ == trans.copy.dst_idx_) {
            data_size += data_size;
        }

        // Get time taken by copy operation. Adjust time from nanoseconds
        // to units of seconds
        if ((print_cpu_time_) || (trans.copy.uses_gpu_ != true)) {
            avg_time = trans.cpu_avg_time_[idx];
            min_time = trans.cpu_min_time_[idx];
            avg_time = avg_time / 1000 / 1000 / 1000;
            min_time = min_time / 1000 / 1000 / 1000;
        } else {
            avg_time = trans.gpu_avg_time_[idx];
            min_time = trans.gpu_min_time_[idx];
        }

        // Determine if there was a validation failure
        // @note: Value is set to VALIDATE_COPY_OP_FAILURE
        // if copy transaction wa validated and it failed
        hsa_status_t verify_status = HSA_STATUS_ERROR;
        if ((avg_time != VALIDATE_COPY_OP_FAILURE) && (min_time != VALIDATE_COPY_OP_FAILURE)) {
            verify_status = HSA_STATUS_SUCCESS;
        }

        // Adjust Gpu time if there is no validation error
        if ((trans.copy.uses_gpu_) && (print_cpu_time_ == false) &&
            (verify_status == HSA_STATUS_SUCCESS)) {
            avg_time = avg_time / sys_freq;
            min_time = min_time / sys_freq;
        }

        // Compute bandwidth - divide bandwidth with
        // 10^9 not 1024^3 to get size in GigaBytes
        // @note: For validation failures bandwidth
        // is encoded by VALIDATE_COPY_OP_FAILURE
        if (verify_status != HSA_STATUS_SUCCESS) {
            avg_bandwidth = VALIDATE_COPY_OP_FAILURE;
            peak_bandwidth = VALIDATE_COPY_OP_FAILURE;
        } else {
            avg_bandwidth = (double)data_size / avg_time / 1000 / 1000 / 1000;
            peak_bandwidth = (double)data_size / min_time / 1000 / 1000 / 1000;
        }

        // Update computed bandwidth for the transaction
        trans.min_time_.push_back(min_time);
        trans.avg_time_.push_back(avg_time);
        trans.avg_bandwidth_.push_back(avg_bandwidth);
        trans.peak_bandwidth_.push_back(peak_bandwidth);
    }
}