From 84816ed2eeeb42f245d0597eb9ff7fca6f540ef3 Mon Sep 17 00:00:00 2001 From: jr Date: Fri, 4 Mar 2022 17:12:51 +0100 Subject: [PATCH 01/12] Explicit list all required clang/llvm libraries as new versions of ld are stricter and don't link libraries just because a symbol in them is referenced. --- cmake/ToolchainOptions.cmake | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmake/ToolchainOptions.cmake b/cmake/ToolchainOptions.cmake index 208a7db..a9c6044 100644 --- a/cmake/ToolchainOptions.cmake +++ b/cmake/ToolchainOptions.cmake @@ -29,7 +29,16 @@ function(add_clang target) # clang specified as system lib to suppress all warnings from clang headers target_include_directories(${target} SYSTEM PUBLIC ${CLANG_INCLUDE_DIRS}) - target_link_libraries(${target} clangTooling) + target_link_libraries( + ${target} + clangTooling + clangFrontend + clangSerialization + clangAST + clangBasic + clangIndex + LLVMSupport + ) endfunction() function(add_extrap target) From f3ef18cf100313dd33d12646690631d200a118fa Mon Sep 17 00:00:00 2001 From: jr Date: Fri, 4 Mar 2022 17:25:34 +0100 Subject: [PATCH 02/12] Check that metacg format 2 is used with custom heuristics --- pgis/tool/PGISMain.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pgis/tool/PGISMain.cpp b/pgis/tool/PGISMain.cpp index ea1e150..91372c3 100644 --- a/pgis/tool/PGISMain.cpp +++ b/pgis/tool/PGISMain.cpp @@ -210,6 +210,13 @@ int main(int argc, char **argv) { checkAndSet(keepUnreachable.cliName, result, keepNotReachable); HeuristicSelection dispose_heuristic; checkAndSet(heuristicSelection.cliName, result, dispose_heuristic); + + if (mcgVersion < 2 && + pgis::config::getSelectedHeuristic() != HeuristicSelection::HeuristicSelectionEnum::STATEMENTS) { + std::cout << "Heuristics other than 'statements' are not supported with metacg format 1" << std::endl; + exit(1); + } + CuttoffSelection dispose_cuttoff; checkAndSet(cuttoffSelection.cliName, result, dispose_cuttoff); From c2156313da1b8fa39020c99359a269c3c045cc68 Mon Sep 17 00:00:00 2001 From: "Kreutzer, Sebastian" Date: Fri, 11 Mar 2022 09:44:54 +0100 Subject: [PATCH 03/12] Read MetaCG version info from first non-null file. closes #40 --- cgcollector/tools/CGMerge.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cgcollector/tools/CGMerge.cpp b/cgcollector/tools/CGMerge.cpp index 42b3ae7..0659af2 100644 --- a/cgcollector/tools/CGMerge.cpp +++ b/cgcollector/tools/CGMerge.cpp @@ -270,8 +270,26 @@ int main(int argc, char **argv) { } nlohmann::json j; - readIPCG(inputFiles.front(), j); - if (j.contains("_CG")) { + bool foundValidFile{false}; + bool useFileFormatTwo{false}; + + for (auto &inFile : inputFiles) { + readIPCG(inFile, j); + if (!j.is_null()) { + foundValidFile = true; + if (j.contains("_CG")) { + useFileFormatTwo = true; + } + break; + } + } + + if (!foundValidFile) { + std::cerr << "[Error] All input files are NULL" << std::endl; + exit(EXIT_FAILURE); + } + + if (useFileFormatTwo) { auto wholeCG = mergeFileFormatTwo(argv[1], inputFiles); writeIPCG(argv[1], wholeCG); } else { From 11cc0bdf8513ada056b9665f59e03fcebff922e7 Mon Sep 17 00:00:00 2001 From: jr Date: Sat, 12 Mar 2022 15:04:32 +0100 Subject: [PATCH 04/12] Use C++17 instead of C++20 We currently don't use any C++20 features and llvm 10 fails to compile with gcc 11 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 474e3e0..4aa895d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ project( set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) From aa2238bdec88ebd6f4893599821a09046b0db480 Mon Sep 17 00:00:00 2001 From: "Heldmann, Tim" Date: Wed, 23 Mar 2022 10:58:38 +0100 Subject: [PATCH 05/12] Simplifies usage of MCGManager / Callgraph interface. - Moves the logic of creating a callgraph to the actual callgraph. - Retains some compatibility by exposing the callgraph's callgraph generation methods via the callgraph manager - Removes some unnecessary computations and enables high cuts in searches to slightly improve performance --- AUTHORS | 2 + graph/include/Callgraph.h | 64 ++++++---- graph/include/MCGManager.h | 24 ++-- graph/src/Callgraph.cpp | 112 +++++++++++------- graph/src/MCGReader.cpp | 5 +- pgis/lib/include/CallgraphManager.h | 2 +- .../include/loadImbalance/LIEstimatorPhase.h | 2 +- pgis/lib/src/CallgraphManager.cpp | 6 +- pgis/lib/src/IPCGEstimatorPhase.cpp | 2 +- pgis/test/unit/CallgraphTest.cpp | 24 ++-- pgis/test/unit/IPCGEstimatorPhaseTest.cpp | 22 ++-- pgis/test/unit/MCGManagerTest.cpp | 10 +- pgis/test/unit/MCGReaderTest.cpp | 8 +- .../loadImbalance/LIEstimatorPhaseTest.cpp | 68 +++++------ 14 files changed, 193 insertions(+), 158 deletions(-) diff --git a/AUTHORS b/AUTHORS index 540bd1d..92548bc 100644 --- a/AUTHORS +++ b/AUTHORS @@ -15,3 +15,5 @@ Constantin Völter Peter Arzt Julian Hindelang Jonas Rickert +Tim Heldmann + diff --git a/graph/include/Callgraph.h b/graph/include/Callgraph.h index 259e12b..57a5152 100644 --- a/graph/include/Callgraph.h +++ b/graph/include/Callgraph.h @@ -15,24 +15,30 @@ namespace metacg { */ class Callgraph { public: - Callgraph() : graph(), mainNode(nullptr), lastSearched(nullptr) {} // TODO: Can this be a hash set? typedef CgNodePtrSet ContainerT; /** - * @brief findMain + * @brief getMain * @return main function CgNodePtr */ - CgNodePtr findMain(); + CgNodePtr getMain(); /** - * @brief findNode - * @param functionName - * @return the corresponding CgNodePtr or nullptr + * Inserts an edge from parentName to childName + * @param parentName function name of calling function + * @param childName function name of called function */ - CgNodePtr findNode(std::string functionName) const; + void addEdge(const std::string &parentName, const std::string &childName); + + /** + * Inserts an edge from parentNode to childNode + * @param parentNode function node of calling function + * @param childNode function node of called function + */ + void addEdge(CgNodePtr parentNode, CgNodePtr childNode); /** * Inserts a new node and sets it as the 'main' function if its name is main or _Z4main or _ZSt4mainiPPc @@ -41,20 +47,17 @@ class Callgraph { void insert(CgNodePtr node); /** - * Clears the graph to an empty graph with no main node and no lastSearched node. + * Returns the node with the given name\n + * If no node exists yet, it creates a new one. + * @param name to identify the node by + * @return CgNodePtr to the identified node */ - void clear() { - graph.clear(); - lastSearched = nullptr; - mainNode = nullptr; - } - - ContainerT::iterator begin() const { return graph.begin(); } - ContainerT::iterator end() const { return graph.end(); } - ContainerT::const_iterator cbegin() const; - ContainerT::const_iterator cend() const; + CgNodePtr getOrInsertNode(const std::string &name); - bool hasNode(CgNodePtr n) { return graph.find(n) != graph.end(); } + /** + * Clears the graph to an empty graph with no main node and no lastSearched node. + */ + void clear(); /** * @brief hasNode checks whether a node for #name exists in the graph mapping @@ -64,10 +67,17 @@ class Callgraph { bool hasNode(std::string name); /** - * @brief getLastSearched - only call after hasNode returned True + * @brief hasNode checks whether a node exists in the graph mapping + * @param node + * @return true iff exists, false otherwise + */ + bool hasNode(CgNodePtr n); + + /** + * @brief getLastSearchedNode - only call after hasNode returned True * @return node found by #hasNode - nullptr otherwise */ - CgNodePtr getLastSearched() { return lastSearched; } + CgNodePtr getLastSearchedNode(); /** * @brief getNode searches the node in the graph and returns it @@ -77,11 +87,15 @@ class Callgraph { CgNodePtr getNode(std::string name); size_t size() const; + ContainerT &getGraph(); - bool isEmpty() { - return graph.size() == 0; - } + bool isEmpty(); + ContainerT::iterator begin() const; + ContainerT::iterator end() const; + + ContainerT::const_iterator cbegin() const; + ContainerT::const_iterator cend() const; private: // this set represents the call graph during the actual computation @@ -90,7 +104,7 @@ class Callgraph { // Dedicated node pointer to main function CgNodePtr mainNode; - // Temporary storage for hasNode/getLastSearched combination + // Temporary storage for hasNode/getLastSearchedNode combination CgNodePtr lastSearched; }; diff --git a/graph/include/MCGManager.h b/graph/include/MCGManager.h index c75d7b1..004cc72 100644 --- a/graph/include/MCGManager.h +++ b/graph/include/MCGManager.h @@ -59,12 +59,10 @@ class MCGManager { * @param parentName function name of calling function * @param childName function name of called function */ + //Note: this is kept for compatibility reasons + // remove this once transition is complete void addEdge(const std::string &parentName, const std::string &childName) { - auto pNode = findOrCreateNode(parentName); - auto cNode = findOrCreateNode(childName); - assert(pNode != nullptr && "Parent node should be in the graph"); - assert(cNode != nullptr && "Child node should be in the graph"); - addEdge(pNode, cNode); + graph.addEdge(parentName, childName); }; /** @@ -72,24 +70,20 @@ class MCGManager { * @param parentNode function node of calling function * @param childNode function node of called function */ + //Note: this is kept for compatibility reasons + // remove this once transition is complete void addEdge(CgNodePtr parentNode, CgNodePtr childNode) { - parentNode->addChildNode(childNode); - childNode->addParentNode(parentNode); + graph.addEdge(parentNode, childNode); } /** * Returns the node for @param name * If no node exists yet, it creates a new one. */ + //Note: this is kept for compatibility reasons + // remove this once transition is complete CgNodePtr findOrCreateNode(const std::string &name) { - auto nodePtr = graph.findNode(name); - if (nodePtr != nullptr) { - return nodePtr; - } - nodePtr = std::make_shared(name); - assert(nodePtr != nullptr && "Creating a new CgNode should work"); - graph.insert(nodePtr); - return nodePtr; + return graph.getOrInsertNode(name); } // Delegates to the underlying graph diff --git a/graph/src/Callgraph.cpp b/graph/src/Callgraph.cpp index 5f54d86..346fb30 100644 --- a/graph/src/Callgraph.cpp +++ b/graph/src/Callgraph.cpp @@ -8,29 +8,25 @@ using namespace metacg; -CgNodePtr Callgraph::findMain() { - if (mainNode) { - return mainNode; - } else { - // simply search a method containing "main" somewhere - for (auto node : *this) { - auto fName = node->getFunctionName(); - if (fName.find("MAIN_") != fName.npos) { - return node; - } - } - return nullptr; - } +CgNodePtr Callgraph::getMain() { + //there is no condition needed + //if a main node exists it was detected on insertion already, and we return it + //if a main node doesn't exist, the default is nullptr, so return that + //revisit this if node deletion is ever supported + return mainNode; } -CgNodePtr Callgraph::findNode(std::string functionName) const { - for (auto node : *this) { - auto fName = node->getFunctionName(); - if (fName == functionName) { - return node; - } - } - return nullptr; +void Callgraph::addEdge(CgNodePtr parentNode, CgNodePtr childNode) { + parentNode->addChildNode(childNode); + childNode->addParentNode(parentNode); +} + +void Callgraph::addEdge(const std::string &parentName, const std::string &childName) { + auto pNode = getOrInsertNode(parentName); + auto cNode = getOrInsertNode(childName); + assert(pNode != nullptr && "Parent node should be in the graph"); + assert(cNode != nullptr && "Child node should be in the graph"); + addEdge(pNode, cNode); } void Callgraph::insert(CgNodePtr node) { @@ -42,36 +38,64 @@ void Callgraph::insert(CgNodePtr node) { graph.insert(node); } -Callgraph::ContainerT::const_iterator Callgraph::cbegin() const { return graph.cbegin(); } -Callgraph::ContainerT::const_iterator Callgraph::cend() const { return graph.cend(); } +CgNodePtr Callgraph::getOrInsertNode(const std::string &name) { + auto nodePtr = getNode(name); + if (nodePtr != nullptr) { + return nodePtr; + } + nodePtr = std::make_shared(name); + assert(nodePtr != nullptr && "Creating a new CgNode should work"); + insert(nodePtr); + return nodePtr; +} -size_t Callgraph::size() const { return graph.size(); } +void Callgraph::clear() { + graph.clear(); + lastSearched = nullptr; + mainNode = nullptr; +} -Callgraph::ContainerT &Callgraph::getGraph() { return graph; } -bool Callgraph::hasNode(std::string name) { - { - if (mainNode && name == "main") { - lastSearched = mainNode; - return true; - } - auto found = std::find_if(graph.begin(), graph.end(), [&](CgNodePtr n) { return n->getFunctionName() == name; }); +bool Callgraph::hasNode(std::string name) { + if (mainNode && name == "main") { + lastSearched = mainNode; + return true; + } - if (found != graph.end()) { - lastSearched = *found; - return true; - } + auto found = std::find_if(graph.begin(), graph.end(), [&](CgNodePtr n) { return n->getFunctionName() == name; }); - lastSearched = nullptr; - return false; + if (found != graph.end()) { + lastSearched = *found; + return true; } + + lastSearched = nullptr; + return false; +} + +bool Callgraph::hasNode(CgNodePtr n) { + return hasNode(n->getFunctionName()); } + +CgNodePtr Callgraph::getLastSearchedNode() { return lastSearched; } + CgNodePtr Callgraph::getNode(std::string name) { - { - auto found = std::find_if(graph.begin(), graph.end(), [&](CgNodePtr n) { return n->getFunctionName() == name; }); - if (found != graph.end()) { - return *found; - } - return nullptr; + if(lastSearched && lastSearched->getFunctionName()==name) + return lastSearched; + + auto found = std::find_if(graph.begin(), graph.end(), [&](CgNodePtr n) { return n->getFunctionName() == name; }); + if (found != graph.end()) { + return *found; } + return nullptr; } +size_t Callgraph::size() const { return graph.size(); } + +bool Callgraph::isEmpty() { return graph.size() == 0; } +Callgraph::ContainerT &Callgraph::getGraph() { return graph; } + +Callgraph::ContainerT::iterator Callgraph::begin() const { return graph.begin(); } +Callgraph::ContainerT::iterator Callgraph::end() const { return graph.end(); } + +Callgraph::ContainerT::const_iterator Callgraph::cbegin() const { return graph.cbegin(); } +Callgraph::ContainerT::const_iterator Callgraph::cend() const { return graph.cend(); } diff --git a/graph/src/MCGReader.cpp b/graph/src/MCGReader.cpp index a960b8b..b59da1e 100644 --- a/graph/src/MCGReader.cpp +++ b/graph/src/MCGReader.cpp @@ -169,7 +169,8 @@ void VersionOneMetaCGReader::read(metacg::graph::MCGManager &cgManager) { // set load imbalance flags in CgNode for (const auto pfi : functions) { - std::optional opt_f = cgManager.getCallgraph().findNode(pfi.first); + std::optional opt_f = cgManager.getCallgraph().getNode(pfi.first); + if (opt_f.has_value()) { CgNodePtr node = opt_f.value(); node->get()->setVirtual(pfi.second.isVirtual); @@ -188,7 +189,7 @@ void VersionOneMetaCGReader::read(metacg::graph::MCGManager &cgManager) { void VersionOneMetaCGReader::addNumStmts(metacg::graph::MCGManager &cgm) { for (const auto &[k, fi] : functions) { auto g = cgm.getCallgraph(); - auto node = g.findNode(fi.functionName); + auto node = g.getNode(fi.functionName); assert(node != nullptr && "Nodes with #statements attached should be available"); if (node->has()){ auto pod = node->get(); diff --git a/pgis/lib/include/CallgraphManager.h b/pgis/lib/include/CallgraphManager.h index 07409c0..b91f7f3 100644 --- a/pgis/lib/include/CallgraphManager.h +++ b/pgis/lib/include/CallgraphManager.h @@ -49,7 +49,7 @@ class PiraMCGProcessor { extrapconnection::ExtrapModelProvider &getModelProvider() { return epModelProvider; } void printMainRuntime() { - const auto mainNode = graph.findMain(); + const auto mainNode = graph.getMain(); const auto inclTime = mainNode->get()->getInclusiveRuntimeInSeconds(); spdlog::get("console")->info("Runtime of main is: {}", inclTime); } diff --git a/pgis/lib/include/loadImbalance/LIEstimatorPhase.h b/pgis/lib/include/loadImbalance/LIEstimatorPhase.h index adda299..2af4d07 100644 --- a/pgis/lib/include/loadImbalance/LIEstimatorPhase.h +++ b/pgis/lib/include/loadImbalance/LIEstimatorPhase.h @@ -27,7 +27,7 @@ class LIEstimatorPhase : public EstimatorPhase { void modifyGraph(CgNodePtr mainMethod) override; - void doPrerequisites() override { CgHelper::calculateInclusiveStatementCounts(graph->findMain()); } + void doPrerequisites() override { CgHelper::calculateInclusiveStatementCounts(graph->getMain()); } private: AbstractMetric *metric; diff --git a/pgis/lib/src/CallgraphManager.cpp b/pgis/lib/src/CallgraphManager.cpp index 3d1be37..c07e198 100644 --- a/pgis/lib/src/CallgraphManager.cpp +++ b/pgis/lib/src/CallgraphManager.cpp @@ -44,7 +44,7 @@ void metacg::pgis::PiraMCGProcessor::finalizeGraph(bool buildMarker) { if (graph.size() > 0) { // We assume that 'main' is always reachable. - auto mainNode = graph.findMain(); + auto mainNode = graph.getMain(); if (mainNode == nullptr) { spdlog::get("errconsole")->error("PiraMCGProcessor: Cannot find main function"); exit(1); @@ -52,7 +52,7 @@ void metacg::pgis::PiraMCGProcessor::finalizeGraph(bool buildMarker) { mainNode->setReachable(); // run reachability analysis -> mark reachable nodes - CgHelper::reachableFromMainAnalysis(graph.findMain()); + CgHelper::reachableFromMainAnalysis(graph.getMain()); // XXX This is a prerequisite for certain EstimatorPhases and should be demanded by them // CgHelper::calculateInclusiveStatementCounts(mainNode); @@ -70,7 +70,7 @@ void metacg::pgis::PiraMCGProcessor::finalizeGraph(bool buildMarker) { void metacg::pgis::PiraMCGProcessor::applyRegisteredPhases() { finalizeGraph(); - auto mainFunction = graph.findMain(); + auto mainFunction = graph.getMain(); if (mainFunction == nullptr) { spdlog::get("errconsole")->error("PiraMCGProcessor: Cannot find main function."); diff --git a/pgis/lib/src/IPCGEstimatorPhase.cpp b/pgis/lib/src/IPCGEstimatorPhase.cpp index b30d82a..9f56f0e 100644 --- a/pgis/lib/src/IPCGEstimatorPhase.cpp +++ b/pgis/lib/src/IPCGEstimatorPhase.cpp @@ -630,7 +630,7 @@ void WLCallpathDifferentiationEstimatorPhase::modifyGraph(CgNodePtr mainMethod) if (line.empty()) { continue; } - CgNodePtr node = graph->findNode(line); + CgNodePtr node = graph->getNode(line); if (node == nullptr) { continue; } diff --git a/pgis/test/unit/CallgraphTest.cpp b/pgis/test/unit/CallgraphTest.cpp index 60d46fe..42fc836 100644 --- a/pgis/test/unit/CallgraphTest.cpp +++ b/pgis/test/unit/CallgraphTest.cpp @@ -20,19 +20,19 @@ class CallgraphTest : public ::testing::Test { TEST_F(CallgraphTest, EmptyCG) { Callgraph c; ASSERT_TRUE(c.isEmpty()); - ASSERT_EQ(nullptr, c.findMain()); + ASSERT_EQ(nullptr, c.getMain()); ASSERT_EQ(0, c.size()); } TEST_F(CallgraphTest, OnlyMainCG) { Callgraph c; ASSERT_TRUE(c.isEmpty()); - ASSERT_EQ(nullptr, c.findMain()); + ASSERT_EQ(nullptr, c.getMain()); auto n = std::make_shared("main"); c.insert(n); ASSERT_FALSE(c.isEmpty()); - ASSERT_EQ(n, c.findMain()); - ASSERT_EQ(n, c.findNode("main")); + ASSERT_EQ(n, c.getMain()); + ASSERT_EQ(n, c.getNode("main")); ASSERT_EQ(n, *(c.begin())); ASSERT_EQ(true, c.hasNode("main")); ASSERT_EQ(1, c.size()); @@ -47,8 +47,8 @@ TEST_F(CallgraphTest, ClearEmptiesGraph) { ASSERT_TRUE(c.hasNode("main")); // sets lastSearched field c.clear(); ASSERT_TRUE(c.isEmpty()); - ASSERT_EQ(nullptr, c.findMain()); - ASSERT_EQ(nullptr, c.getLastSearched()); + ASSERT_EQ(nullptr, c.getMain()); + ASSERT_EQ(nullptr, c.getLastSearchedNode()); } TEST_F(CallgraphTest, TwoNodeConnectedCG) { @@ -62,8 +62,8 @@ TEST_F(CallgraphTest, TwoNodeConnectedCG) { ASSERT_EQ(2, c.size()); ASSERT_EQ(true, c.hasNode("main")); ASSERT_EQ(true, c.hasNode("child")); - ASSERT_EQ(main, c.findMain()); - auto founMain = c.findMain(); + ASSERT_EQ(main, c.getMain()); + auto founMain = c.getMain(); ASSERT_EQ(child, (*founMain->getChildNodes().begin())); } @@ -71,9 +71,9 @@ TEST_F(CallgraphTest, HasNodeGetLastSearchedTest) { Callgraph c; auto main = std::make_shared("child"); c.insert(main); - ASSERT_EQ(nullptr, c.getLastSearched()); + ASSERT_EQ(nullptr, c.getLastSearchedNode()); c.hasNode("child"); - ASSERT_EQ(main, c.getLastSearched()); + ASSERT_EQ(main, c.getLastSearchedNode()); } TEST_F(CallgraphTest, InsertTwiceTest) { @@ -93,8 +93,8 @@ TEST_F(CallgraphTest, SearchNodes) { node2->addChildNode(node); node->addParentNode(node2); c.insert(node2); - ASSERT_EQ(nullptr, c.findMain()); + ASSERT_EQ(nullptr, c.getMain()); ASSERT_EQ(false, c.hasNode("main")); ASSERT_EQ(false, c.hasNode("nodeee")); - ASSERT_EQ(nullptr, c.getLastSearched()); + ASSERT_EQ(nullptr, c.getLastSearchedNode()); } diff --git a/pgis/test/unit/IPCGEstimatorPhaseTest.cpp b/pgis/test/unit/IPCGEstimatorPhaseTest.cpp index 30af3d5..5da4d28 100644 --- a/pgis/test/unit/IPCGEstimatorPhaseTest.cpp +++ b/pgis/test/unit/IPCGEstimatorPhaseTest.cpp @@ -53,7 +53,7 @@ TEST_F(IPCGEstimatorPhaseBasic, OneNodeCG) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = mcgm.getCallgraph(); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(0, sce.getNumStatements(mainNode)); cm.removeAllEstimatorPhases(); } @@ -74,7 +74,7 @@ TEST_F(IPCGEstimatorPhaseBasic, TwoNodeCG) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(0, sce.getNumStatements(mainNode)); cm.removeAllEstimatorPhases(); } @@ -100,7 +100,7 @@ TEST_F(IPCGEstimatorPhaseBasic, OneNodeCGwStmt) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(12, sce.getNumStatements(mainNode)); cm.removeAllEstimatorPhases(); } @@ -127,7 +127,7 @@ TEST_F(IPCGEstimatorPhaseBasic, TwoNodeCGwStmt) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(19, sce.getNumStatements(mainNode)); ASSERT_EQ(7, sce.getNumStatements(childNode)); cm.removeAllEstimatorPhases(); @@ -161,7 +161,7 @@ TEST_F(IPCGEstimatorPhaseBasic, ThreeNodeCGwStmt) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(61, sce.getNumStatements(mainNode)); ASSERT_EQ(7, sce.getNumStatements(childNode)); ASSERT_EQ(42, sce.getNumStatements(childNode2)); @@ -196,7 +196,7 @@ TEST_F(IPCGEstimatorPhaseBasic, ThreeNodeCycleCGwStmt) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(61, sce.getNumStatements(mainNode)); ASSERT_EQ(49, sce.getNumStatements(childNode)); ASSERT_EQ(49, sce.getNumStatements(childNode2)); @@ -236,7 +236,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FourNodeCGwStmt) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(83, sce.getNumStatements(mainNode)); ASSERT_EQ(29, sce.getNumStatements(childNode)); ASSERT_EQ(42, sce.getNumStatements(childNode2)); @@ -277,7 +277,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FourNodeDiamondCGwStmt) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(105, sce.getNumStatements(mainNode)); ASSERT_EQ(29, sce.getNumStatements(childNode)); ASSERT_EQ(64, sce.getNumStatements(childNode2)); @@ -332,7 +332,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FiveNodeDiamondCGwStmt) { cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(mainNode, graph.findMain()); + ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(113, sce.getNumStatements(mainNode)); ASSERT_EQ(33, sce.getNumStatements(childNode)); ASSERT_EQ(68, sce.getNumStatements(childNode2)); @@ -387,9 +387,9 @@ class IPCGEstimatorPhaseTest : public ::testing::Test { TEST_F(IPCGEstimatorPhaseTest, ValidateBasics) { auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto graph = cm.getCallgraph(&cm); - ASSERT_NE(nullptr, graph.findMain()); + ASSERT_NE(nullptr, graph.getMain()); - auto mainNode = graph.findMain(); + auto mainNode = graph.getMain(); EXPECT_EQ(3, mainNode->getChildNodes().size()); auto childNodes = mainNode->getChildNodes(); auto nodeIter = childNodes.begin(); diff --git a/pgis/test/unit/MCGManagerTest.cpp b/pgis/test/unit/MCGManagerTest.cpp index befd56d..23e86aa 100644 --- a/pgis/test/unit/MCGManagerTest.cpp +++ b/pgis/test/unit/MCGManagerTest.cpp @@ -28,7 +28,7 @@ TEST_F(MCGManagerTest, EmptyCG) { auto graph = mcgm.getCallgraph(); ASSERT_TRUE(graph.isEmpty()); ASSERT_EQ(false, graph.hasNode("main")); - ASSERT_EQ(nullptr, graph.findMain()); + ASSERT_EQ(nullptr, graph.getMain()); ASSERT_EQ(0, graph.size()); } @@ -38,8 +38,8 @@ TEST_F(MCGManagerTest, OneNodeCG) { auto nPtr = mcgm.findOrCreateNode("main"); auto graph = mcgm.getCallgraph(); ASSERT_FALSE(graph.isEmpty()); - ASSERT_NE(nullptr, graph.findMain()); - ASSERT_EQ(nPtr, graph.findMain()); + ASSERT_NE(nullptr, graph.getMain()); + ASSERT_EQ(nPtr, graph.getMain()); } TEST_F(MCGManagerTest, TwoNodeCG) { @@ -50,8 +50,8 @@ TEST_F(MCGManagerTest, TwoNodeCG) { ASSERT_EQ(mainNode, mcgm.findOrCreateNode("main")); ASSERT_EQ(childNode, mcgm.findOrCreateNode("child1")); auto graph = mcgm.getCallgraph(); - ASSERT_EQ(mainNode, graph.findMain()); - ASSERT_EQ(childNode, graph.findNode("child1")); + ASSERT_EQ(mainNode, graph.getMain()); + ASSERT_EQ(childNode, graph.getNode("child1")); } TEST_F(MCGManagerTest, ThreeNodeCG) { diff --git a/pgis/test/unit/MCGReaderTest.cpp b/pgis/test/unit/MCGReaderTest.cpp index cdc9e1a..f6e19e8 100644 --- a/pgis/test/unit/MCGReaderTest.cpp +++ b/pgis/test/unit/MCGReaderTest.cpp @@ -215,7 +215,7 @@ TEST(VersionTwoMetaCGReaderTest, OneNodeNoMetaDataHandler) { auto graph = mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 1); - const auto mainNode = graph.findMain(); + const auto mainNode = graph.getMain(); EXPECT_EQ(mainNode->getFunctionName(), "main"); EXPECT_EQ(mainNode->getChildNodes().size(), 0); EXPECT_EQ(mainNode->getParentNodes().size(), 0); @@ -259,7 +259,7 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesNoMetaDataHandler) { auto graph = mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 2); - const auto mainNode = graph.findMain(); + const auto mainNode = graph.getMain(); EXPECT_EQ(mainNode->getFunctionName(), "main"); EXPECT_EQ(mainNode->getChildNodes().size(), 1); EXPECT_EQ(mainNode->getParentNodes().size(), 0); @@ -304,7 +304,7 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesOneMetaDataHandler) { auto graph = mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 2); - const auto mainNode = graph.findMain(); + const auto mainNode = graph.getMain(); EXPECT_EQ(mainNode->getFunctionName(), "main"); EXPECT_EQ(mainNode->getChildNodes().size(), 1); EXPECT_EQ(mainNode->getParentNodes().size(), 0); @@ -365,7 +365,7 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesTwoMetaDataHandler) { auto graph = mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 2); - const auto mainNode = graph.findMain(); + const auto mainNode = graph.getMain(); EXPECT_EQ(mainNode->getFunctionName(), "main"); EXPECT_EQ(mainNode->getChildNodes().size(), 1); EXPECT_EQ(mainNode->getParentNodes().size(), 0); diff --git a/pgis/test/unit/loadImbalance/LIEstimatorPhaseTest.cpp b/pgis/test/unit/loadImbalance/LIEstimatorPhaseTest.cpp index 141991c..fc04b9d 100644 --- a/pgis/test/unit/loadImbalance/LIEstimatorPhaseTest.cpp +++ b/pgis/test/unit/loadImbalance/LIEstimatorPhaseTest.cpp @@ -132,20 +132,20 @@ TEST_F(LIEstimatorPhaseTest, AllCases) { auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(graph.findMain(), mainNode); - - ASSERT_EQ(graph.findMain()->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child1")->isInstrumented(), false); - ASSERT_EQ(graph.findNode("child2")->isInstrumented(), false); - ASSERT_EQ(graph.findNode("child3")->isInstrumented(), false); - ASSERT_EQ(graph.findNode("child4")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child5")->isInstrumented(), false); - - ASSERT_EQ(graph.findNode("gc1")->isInstrumented(), false); - ASSERT_EQ(graph.findNode("gc2")->isInstrumented(), false); - ASSERT_EQ(graph.findNode("gc3")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("gc4")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("gc5")->isInstrumented(), true); + ASSERT_EQ(graph.getMain(), mainNode); + + ASSERT_EQ(graph.getMain()->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child1")->isInstrumented(), false); + ASSERT_EQ(graph.getNode("child2")->isInstrumented(), false); + ASSERT_EQ(graph.getNode("child3")->isInstrumented(), false); + ASSERT_EQ(graph.getNode("child4")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child5")->isInstrumented(), false); + + ASSERT_EQ(graph.getNode("gc1")->isInstrumented(), false); + ASSERT_EQ(graph.getNode("gc2")->isInstrumented(), false); + ASSERT_EQ(graph.getNode("gc3")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("gc4")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("gc5")->isInstrumented(), true); } TEST_F(LIEstimatorPhaseTest, Virtual) { @@ -193,11 +193,11 @@ TEST_F(LIEstimatorPhaseTest, Virtual) { auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(graph.findMain(), mainNode); - ASSERT_EQ(graph.findNode("main")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("grandchild")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("grandgrandchild")->isInstrumented(), false); + ASSERT_EQ(graph.getMain(), mainNode); + ASSERT_EQ(graph.getNode("main")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("grandchild")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("grandgrandchild")->isInstrumented(), false); } TEST_F(LIEstimatorPhaseTest, AllPathsToMain) { @@ -250,11 +250,11 @@ TEST_F(LIEstimatorPhaseTest, AllPathsToMain) { auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(graph.findMain(), mainNode); - ASSERT_EQ(graph.findNode("main")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child1")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child2")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("grandchild")->isInstrumented(), true); + ASSERT_EQ(graph.getMain(), mainNode); + ASSERT_EQ(graph.getNode("main")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child1")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child2")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("grandchild")->isInstrumented(), true); } TEST_F(LIEstimatorPhaseTest, MajorPathsToMain) { @@ -306,11 +306,11 @@ TEST_F(LIEstimatorPhaseTest, MajorPathsToMain) { auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(graph.findMain(), mainNode); - ASSERT_EQ(graph.findNode("main")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child1")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child2")->isInstrumented(), false); - ASSERT_EQ(graph.findNode("grandchild")->isInstrumented(), true); + ASSERT_EQ(graph.getMain(), mainNode); + ASSERT_EQ(graph.getNode("main")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child1")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child2")->isInstrumented(), false); + ASSERT_EQ(graph.getNode("grandchild")->isInstrumented(), true); } TEST_F(LIEstimatorPhaseTest, MajorParentSteps) { @@ -357,9 +357,9 @@ TEST_F(LIEstimatorPhaseTest, MajorParentSteps) { auto graph = cm.getCallgraph(&cm); - ASSERT_EQ(graph.findMain(), mainNode); - ASSERT_EQ(graph.findNode("main")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child1")->isInstrumented(), false); - ASSERT_EQ(graph.findNode("child2")->isInstrumented(), true); - ASSERT_EQ(graph.findNode("child3")->isInstrumented(), true); + ASSERT_EQ(graph.getMain(), mainNode); + ASSERT_EQ(graph.getNode("main")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child1")->isInstrumented(), false); + ASSERT_EQ(graph.getNode("child2")->isInstrumented(), true); + ASSERT_EQ(graph.getNode("child3")->isInstrumented(), true); } From b626984f8182e81d073d7748ef71235dbe43966f Mon Sep 17 00:00:00 2001 From: "Lehr, Jan-Patrick" Date: Thu, 24 Mar 2022 23:15:09 +0100 Subject: [PATCH 06/12] Implements MCGWriter in graph library - Implements `value` method of MetaDataHandlers for existing MetaData - Adds initial unit tests for three commonly used meta data --- CMakeLists.txt | 15 ++ cgcollector/Config.h.in => Config.h.in | 0 cgcollector/CMakeLists.txt | 17 -- cmake/ToolchainOptions.cmake | 9 +- graph/CMakeLists.txt | 5 + graph/include/CgNode.h | 2 +- graph/include/MCGManager.h | 2 +- graph/include/MCGReader.h | 50 ----- graph/include/MCGWriter.h | 134 ++++++++++++ graph/include/MetaData.h | 2 +- graph/include/Util.h | 26 +++ graph/src/MCGWriter.cpp | 85 ++++++++ pgis/lib/include/CgNodeMetaData.h | 54 +---- pgis/lib/include/Utility.h | 31 +++ pgis/lib/include/libIPCG/MetaDataHandler.h | 42 +++- pgis/lib/include/loadImbalance/LIRetriever.h | 3 +- pgis/lib/src/libIPCG/MetaDataHandler.cpp | 114 ++++++++++- pgis/lib/src/loadImbalance/LIRetriever.cpp | 6 + pgis/test/unit/CMakeLists.txt | 1 + pgis/test/unit/IPCGAnnotation.cpp | 2 +- pgis/test/unit/MCGReaderTest.cpp | 4 +- pgis/test/unit/MCGWriterTest.cpp | 192 ++++++++++++++++++ .../unit/loadImbalance/LIMetaDataTest.cpp | 2 +- pgis/tool/PGISMain.cpp | 14 +- 24 files changed, 670 insertions(+), 142 deletions(-) rename cgcollector/Config.h.in => Config.h.in (100%) create mode 100644 graph/include/MCGWriter.h create mode 100644 graph/include/Util.h create mode 100644 graph/src/MCGWriter.cpp create mode 100644 pgis/test/unit/MCGWriterTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4aa895d..90f2f94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,21 @@ include(CMakePackageConfigHelpers) # Making printing / debugging easier include(CMakePrintHelpers) +# Obtain ghe git revision of this MetaCG repository +include(GetGitRevisionDescription) + +# Stream the verison info in the project's config.h +get_git_head_revision(GIT_REFSPEC GIT_SHA1) +if(NOT + DEFINED + GIT_SHA1 +) + set(GIT_SHA1 "NO_GIT_ROOT") +endif() +message(${GIT_SHA1}) + +configure_file(Config.h.in config.h) + # Component options MetaCG graph library will always be built. The actual graph implementation add_subdirectory(graph) diff --git a/cgcollector/Config.h.in b/Config.h.in similarity index 100% rename from cgcollector/Config.h.in rename to Config.h.in diff --git a/cgcollector/CMakeLists.txt b/cgcollector/CMakeLists.txt index 3463b24..6660e86 100644 --- a/cgcollector/CMakeLists.txt +++ b/cgcollector/CMakeLists.txt @@ -1,10 +1,6 @@ set(PROJECT_NAME CGCollector) set(TARGETS_EXPORT_NAME ${PROJECT_NAME}-target) -function(add_config_include target) - target_include_directories(${target} PUBLIC "${PROJECT_BINARY_DIR}") -endfunction() - # include directories function(add_collector_include target) target_include_directories( @@ -49,19 +45,6 @@ function(default_compile_options target) endif() endfunction() -include(GetGitRevisionDescription) - -get_git_head_revision(GIT_REFSPEC GIT_SHA1) -if(NOT - DEFINED - GIT_SHA1 -) - set(GIT_SHA1 "NO_GIT_ROOT") -endif() -message(${GIT_SHA1}) - -configure_file(Config.h.in config.h) - add_subdirectory(lib) add_subdirectory(tools) add_subdirectory(test) diff --git a/cmake/ToolchainOptions.cmake b/cmake/ToolchainOptions.cmake index a9c6044..e30e72c 100644 --- a/cmake/ToolchainOptions.cmake +++ b/cmake/ToolchainOptions.cmake @@ -81,8 +81,15 @@ function(add_mcg target) add_mcg_library(${target}) endfunction() +function(add_config_include target) + target_include_directories(${target} PUBLIC "${PROJECT_BINARY_DIR}") +endfunction() + function(add_pgis_includes target) - target_include_directories(${target} PUBLIC $) + target_include_directories( + ${target} PUBLIC $ + $ + ) endfunction() function(add_graph_includes target) diff --git a/graph/CMakeLists.txt b/graph/CMakeLists.txt index f0abcbd..3c95a15 100644 --- a/graph/CMakeLists.txt +++ b/graph/CMakeLists.txt @@ -12,6 +12,9 @@ set(MetacgGraphLibSources include/CgNodePtr.h include/MetaData.h include/MCGManager.h + include/MCGWriter.h + src/MCGWriter.cpp + include/Util.h ) add_library(metacg SHARED ${MetacgGraphLibSources}) @@ -22,6 +25,8 @@ target_include_directories(metacg PUBLIC $) +add_config_include(metacg) + add_json(metacg) add_spdlog_libraries(metacg) add_extrap(metacg) diff --git a/graph/include/CgNode.h b/graph/include/CgNode.h index 8462809..1947bc4 100644 --- a/graph/include/CgNode.h +++ b/graph/include/CgNode.h @@ -70,7 +70,7 @@ class CgNode { */ template inline T *get() const { - assert(metaFields[T::key()] && "meta field for key exists"); + assert(metaFields[T::key()] && "meta field for key must exist"); auto val = metaFields[T::key()]; return reinterpret_cast(val); } diff --git a/graph/include/MCGManager.h b/graph/include/MCGManager.h index 004cc72..2a6b2dc 100644 --- a/graph/include/MCGManager.h +++ b/graph/include/MCGManager.h @@ -46,7 +46,7 @@ class MCGManager { * Returns list of non-owned pointers to MetaDataHandler. * @return */ - std::vector getMetaHandlers() { + std::vector getMetaHandlers() const { std::vector handler; for (const auto &mh : metaHandlers) { handler.push_back(mh.get()); diff --git a/graph/include/MCGReader.h b/graph/include/MCGReader.h index 305d8b1..f6bdcbd 100644 --- a/graph/include/MCGReader.h +++ b/graph/include/MCGReader.h @@ -168,56 +168,6 @@ class VersionTwoMetaCGReader : public MetaCGReader { void read(metacg::graph::MCGManager &cgManager) override; }; -/* - * - * (Old?) Annotation mechanism - * - */ - -template -int doAnnotate(metacg::Callgraph &cg, PropRetriever retriever, json &j) { - const auto functionElement = [](json &container, auto name) { - for (json::iterator it = container.begin(); it != container.end(); ++it) { - if (it.key() == name) { - return it; - } - } - return container.end(); - }; - - const auto holdsValue = [](auto jsonIt, auto jsonEnd) { return jsonIt != jsonEnd; }; - - int annots = 0; - for (const auto &node : cg) { - if (retriever.handles(node)) { - auto funcElem = functionElement(j, node->getFunctionName()); - - if (holdsValue(funcElem, j.end())) { - auto &nodeMeta = (*funcElem)["meta"]; - nodeMeta[retriever.toolName()] = retriever.value(node); - annots++; - } - } - } - return annots; -} - -template -void annotateJSON(metacg::Callgraph &cg, const std::string &filename, PropRetriever retriever) { - json j; - { - std::ifstream in(filename); - in >> j; - } - - int annotated = metacg::io::doAnnotate(cg, retriever, j); - spdlog::get("console")->trace("Annotated {} json nodes", annotated); - - { - std::ofstream out(filename); - out << j << std::endl; - } -} diff --git a/graph/include/MCGWriter.h b/graph/include/MCGWriter.h new file mode 100644 index 0000000..0880eca --- /dev/null +++ b/graph/include/MCGWriter.h @@ -0,0 +1,134 @@ +/** +* File: MCGWriter.h +* License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at +* https://github.com/tudasc/metacg/LICENSE.txt +*/ + +#ifndef METACG_MCGWRITER_H +#define METACG_MCGWRITER_H + +#include "config.h" +#include "Callgraph.h" + +#include "nlohmann/json.hpp" + +namespace metacg::io { + +/** + * Target sink to serialize the CG into a json object. + */ +class JsonSink { + public: + explicit JsonSink() {} + + void setJson(nlohmann::json jsonIn) { + j = jsonIn; + } + + [[nodiscard]] const nlohmann::json & getJson() const { + return j; + } + + /** + * Outputs the Json stored in this sink into os and flushes. + * @param os + */ + void output(std::ostream &os) { + os << j; + os.flush(); + } + + private: + nlohmann::json j; +}; + +/** + * Class to serialize the CG. + */ +class MCGWriter { + public: + explicit MCGWriter(graph::MCGManager &mcgm) : mcgManager(mcgm) {} + + void write(JsonSink &js); + + private: + /** + * Adds the CG version data to the MetaCG in json Format. + * @param j + */ + inline void attachFormatTwoHeader(nlohmann::json &j) { + std::string cgcMajorVersion = std::to_string(CGCollector_VERSION_MAJOR); + std::string cgcMinorVersion = std::to_string(CGCollector_VERSION_MINOR); + std::string cgcVersion{cgcMajorVersion + '.' + cgcMinorVersion}; + j = {{"_MetaCG", {}}, {"_CG", {}}}; + j["_MetaCG"] = {{"version", "2.0"}, + {"generator", {{"name", "CGCollector"}, {"version", cgcVersion}, {"sha", MetaCG_GIT_SHA}}}}; + } + + /** + * General construction of node data, e.g., function name. + * @param node + */ + void createNodeData(const CgNodePtr node, nlohmann::json &j); + + /** + * Using the stored MetaDataHandler in MCGM to walk MCG and extract all meta data. + * @param node + * @param mcgm + */ + void createAndAddMetaData(CgNodePtr node, const graph::MCGManager &mcgm, nlohmann::json &j); + + graph::MCGManager &mcgManager; +}; + +/* + * Old Annotation mechanism. + * TODO: Replace fully with MCGWriter mechanism. + */ +template +int doAnnotate(metacg::Callgraph &cg, PropRetriever retriever, nlohmann::json &j) { + const auto functionElement = [](nlohmann::json &container, auto name) { + for (nlohmann::json::iterator it = container.begin(); it != container.end(); ++it) { + if (it.key() == name) { + return it; + } + } + return container.end(); + }; + + const auto holdsValue = [](auto jsonIt, auto jsonEnd) { return jsonIt != jsonEnd; }; + + int annots = 0; + for (const auto &node : cg) { + if (retriever.handles(node)) { + auto funcElem = functionElement(j, node->getFunctionName()); + + if (holdsValue(funcElem, j.end())) { + auto &nodeMeta = (*funcElem)["meta"]; + nodeMeta[retriever.toolName()] = retriever.value(node); + annots++; + } + } + } + return annots; +} + +template +void annotateJSON(metacg::Callgraph &cg, const std::string &filename, PropRetriever retriever) { + nlohmann::json j; + { + std::ifstream in(filename); + in >> j; + } + + int annotated = metacg::io::doAnnotate(cg, retriever, j); + spdlog::get("console")->trace("Annotated {} json nodes", annotated); + + { + std::ofstream out(filename); + out << j << std::endl; + } +} + +} // namespace metacg::io +#endif // METACG_MCGWRITER_H diff --git a/graph/include/MetaData.h b/graph/include/MetaData.h index 0ab6517..4393d23 100644 --- a/graph/include/MetaData.h +++ b/graph/include/MetaData.h @@ -43,7 +43,7 @@ struct MetaDataHandler { [[nodiscard]] virtual const std::string toolName() const = 0; /** Creates or returns the object to attach as meta information */ - const std::string value(const CgNodePtr n) const { return "Should not have happened"; } + virtual json value(const CgNodePtr n) const { return "Should not have happened"; } /** Reads the meta data from the json file and attaches it to the graph nodes */ virtual void read([[maybe_unused]] const json &j, const std::string &functionName) = 0; diff --git a/graph/include/Util.h b/graph/include/Util.h new file mode 100644 index 0000000..7847b96 --- /dev/null +++ b/graph/include/Util.h @@ -0,0 +1,26 @@ +/** +* File: Util.h +* License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at +* https://github.com/tudasc/metacg/LICENSE.txt +*/ + +#ifndef METACG_UTIL_H +#define METACG_UTIL_H + +#include "CgNode.h" +#include +#include + +namespace metacg::util { + +std::set to_string(const std::set &nodes) { + std::set names; + for (const auto n : nodes) { + names.emplace(n->getFunctionName()); + } + return names; +} + +} + +#endif // METACG_UTIL_H diff --git a/graph/src/MCGWriter.cpp b/graph/src/MCGWriter.cpp new file mode 100644 index 0000000..8c32835 --- /dev/null +++ b/graph/src/MCGWriter.cpp @@ -0,0 +1,85 @@ +/** + * File: MCGWriter.cpp + * License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at + * https://github.com/tudasc/metacg/LICENSE.txt + */ + +#include "MCGWriter.h" +#include "MCGManager.h" +#include "Util.h" + +using FunctionNames = std::set; + +//inline void insertNode(nlohmann::json &callgraph, const std::string &nodeName, const FunctionNames &callees, +// const FunctionNames &callers, const FunctionNames &overriddenBy, +// const FunctionNames &overriddenFunctions, const bool isVirtual, const bool doesOverride, +// const bool hasBody, int version) { +// if (version == 1) { +// callgraph[nodeName] = {{"callees", callees}, +// {"isVirtual", isVirtual}, +// {"doesOverride", doesOverride}, +// {"overriddenFunctions", overriddenFunctions}, +// {"overriddenBy", overriddenBy}, +// {"parents", callers}, +// {"hasBody", hasBody}}; +// } else if (version == 2) { +// callgraph["_CG"][nodeName] = {{"callees", callees}, +// {"isVirtual", isVirtual}, +// {"doesOverride", doesOverride}, +// {"overrides", overriddenFunctions}, +// {"overriddenBy", overriddenBy}, +// {"callers", callers}, +// {"hasBody", hasBody}}; +// } +//} + +void metacg::io::MCGWriter::write(JsonSink &js) { + nlohmann::json j; + attachFormatTwoHeader(j); + + for (const auto &n : mcgManager.getCallgraph()) { + createNodeData(n, j); // general node data? + createAndAddMetaData(n, mcgManager, j); + } + + js.setJson(j); +} + +void metacg::io::MCGWriter::createNodeData(const CgNodePtr node, nlohmann::json &j) { + // Currently correctly stored in CgNode + const auto funcName = node->getFunctionName(); + const auto isVirtual = node->isVirtual(); + const auto callees = node->getChildNodes(); + const auto callers = node->getParentNodes(); + const auto hasBody = node->getHasBody(); + + // XXX Currently, this information is lost when building the in-memory metacg + // const auto overrides = node->getOverrides(); + // const auto overriddenBy = node->getOverriddenBy(); + // const auto doesOverride = node->getDoesOverride(); + const bool doesOverride {false}; + const std::set overrides; + const std::set overriddenBy; +// insertNode(j, funcName, callees, callers, overriddenBy, overrides, isVirtual, doesOverride, hasBody, 2); + + j["_CG"][funcName] = {{"callees", metacg::util::to_string(callees)}, + {"isVirtual", isVirtual}, + {"doesOverride", doesOverride}, + {"overrides", overrides}, + {"overriddenBy", overriddenBy}, + {"callers", metacg::util::to_string(callers)}, + {"hasBody", hasBody}, + {"meta", nullptr}}; +} + +void metacg::io::MCGWriter::createAndAddMetaData(CgNodePtr node, const metacg::graph::MCGManager &mcgm, nlohmann::json &j) { + const auto funcName = node->getFunctionName(); + const auto mdHandlers = mcgm.getMetaHandlers(); + for (const auto mdh : mdHandlers) { + if (!mdh->handles(node)) { + continue; + } + const auto mdJson = mdh->value(node); + j["_CG"][funcName]["meta"][mdh->toolName()] = mdJson; + } +} diff --git a/pgis/lib/include/CgNodeMetaData.h b/pgis/lib/include/CgNodeMetaData.h index 93a7650..2d325c6 100644 --- a/pgis/lib/include/CgNodeMetaData.h +++ b/pgis/lib/include/CgNodeMetaData.h @@ -82,9 +82,6 @@ class BaseProfileData : public metacg::MetaData { std::vector cgLoc; }; -inline void to_json(nlohmann::json &j, const BaseProfileData &data) { - j = nlohmann::json{{"numCalls", data.getNumberOfCalls()}, {"timeInSeconds", data.getRuntimeInSeconds()}}; -} /** * This class holds data relevant to the PIRA I analyses. @@ -208,57 +205,8 @@ class GlobalLoopDepthMetaData : public metacg::MetaData { int globalLoopDepth{0}; }; -/** - * TODO This works for only a single parameter for now! - */ -template -auto valTup(C1 co, C2 ct, int numReps) { - // TODO This assert seems wrong - // assert(co.size() == ct.size() && "Can only value-tuple evenly sized containers"); - std::vector< - std::pair>> - res; - if (ct.empty()) { - return res; - } - assert(ct.size() == 1 && "Current limitation, only single parameter possible"); - auto coIt = std::begin(co); - // Compute the median per numReps from co first. - const auto median = [&](auto startIt, auto numElems) { - if (numElems % 2 == 0) { - return startIt[(numElems / 2) + 1]; - } - return startIt[numElems / 2]; - }; - auto innerC = ct[0].second; - auto ctIt = std::begin(innerC); - res.reserve(co.size()); - for (; coIt != co.end() && ctIt != innerC.end(); std::advance(coIt, numReps), ++ctIt) { - res.push_back(std::make_pair(median(coIt, numReps), std::make_pair(ct[0].first, *ctIt))); - } - return res; -} -inline void to_json(nlohmann::json &j, const PiraTwoData &data) { - auto &gOpts = pgis::config::GlobalConfig::get(); - auto rtOnly = gOpts.getAs("runtime-only"); - - auto rtAndParams = valTup(data.getRuntimeVec(), data.getExtrapParameters(), data.getNumReps()); - nlohmann::json experiments; - for (auto elem : rtAndParams) { - nlohmann::json exp{}; - exp["runtime"] = elem.first; - exp[elem.second.first] = elem.second.second; - experiments += exp; - } - if (!rtOnly) { - j = nlohmann::json{{"experiments", experiments}, - {"model", data.getExtrapModel()->getAsString(data.getExtrapModelConnector().getParamList())}}; - } else { - j = nlohmann::json{{"experiments", experiments}}; - } - spdlog::get("console")->debug("PiraTwoData to_json:\n{}", j.dump()); -} + } // namespace pira diff --git a/pgis/lib/include/Utility.h b/pgis/lib/include/Utility.h index 7874e10..4d2d963 100644 --- a/pgis/lib/include/Utility.h +++ b/pgis/lib/include/Utility.h @@ -27,6 +27,37 @@ inline std::string getHostname() { free(cName); return name; } + +/** + * TODO This works for only a single parameter for now! + */ +template +auto valTup(C1 co, C2 ct, int numReps) { + // TODO This assert seems wrong + // assert(co.size() == ct.size() && "Can only value-tuple evenly sized containers"); + std::vector< + std::pair>> + res; + if (ct.empty()) { + return res; + } + assert(ct.size() == 1 && "Current limitation, only single parameter possible"); + auto coIt = std::begin(co); + // Compute the median per numReps from co first. + const auto median = [&](auto startIt, auto numElems) { + if (numElems % 2 == 0) { + return startIt[(numElems / 2) + 1]; + } + return startIt[numElems / 2]; + }; + auto innerC = ct[0].second; + auto ctIt = std::begin(innerC); + res.reserve(co.size()); + for (; coIt != co.end() && ctIt != innerC.end(); std::advance(coIt, numReps), ++ctIt) { + res.push_back(std::make_pair(median(coIt, numReps), std::make_pair(ct[0].first, *ctIt))); + } + return res; +} } // namespace #endif diff --git a/pgis/lib/include/libIPCG/MetaDataHandler.h b/pgis/lib/include/libIPCG/MetaDataHandler.h index 3417fe6..5bdfc2b 100644 --- a/pgis/lib/include/libIPCG/MetaDataHandler.h +++ b/pgis/lib/include/libIPCG/MetaDataHandler.h @@ -16,6 +16,7 @@ #include "Utility.h" #include "CallgraphManager.h" #include "nlohmann/json.hpp" +#include "GlobalConfig.h" // System library #include @@ -40,7 +41,18 @@ struct TestHandler : public MetaDataHandler { const std::string toolName() const override { return "TestMetaHandler"; } void read([[maybe_unused]] const json &j, const std::string &functionName) override { i++; } bool handles(const CgNodePtr n) const override { return false; } - int value(const CgNodePtr n) const { return i; } + json value(const CgNodePtr n) const override { + json j; + return j << i; + } +}; + +struct BaseProfileDataHandler : public MetaDataHandler { + const std::string toolname{"baseProfileData"}; + bool handles(const CgNodePtr n) const override { return n->has(); } + const std::string toolName() const override { return toolname; } + json value(const CgNodePtr n) const override; + void read([[maybe_unused]] const json &j, const std::string &functionName) override; }; /** @@ -48,9 +60,13 @@ struct TestHandler : public MetaDataHandler { */ struct PiraOneDataRetriever : public MetaDataHandler { const std::string toolname{"numStatements"}; - bool handles(const CgNodePtr n) const override { return false; } + bool handles(const CgNodePtr n) const override { return n->has(); } const std::string toolName() const override { return toolname; } - int value(const CgNodePtr n) const { return 42; } + json value(const CgNodePtr n) const override { + json j; + j["numStatements"] = n->get()->getNumberOfStatements(); + return j; + } void read([[maybe_unused]] const json &j, const std::string &functionName) override; }; @@ -60,7 +76,7 @@ struct PiraOneDataRetriever : public MetaDataHandler { struct PiraTwoDataRetriever : public MetaDataHandler { const std::string toolname{"PiraIIData"}; bool handles(const CgNodePtr n) const override; - pira::PiraTwoData value(const CgNodePtr n) const; + json value(const CgNodePtr n) const override; const std::string toolName() const override { return toolname; } void read([[maybe_unused]] const json &j, const std::string &functionName) override; }; @@ -70,44 +86,50 @@ struct PiraTwoDataRetriever : public MetaDataHandler { */ struct FilePropertyHandler : public MetaDataHandler { const std::string toolname{"fileProperties"}; - bool handles(const CgNodePtr n) const override { return true; } + bool handles(const CgNodePtr n) const override { return n->has(); } const std::string toolName() const override { return toolname; } void read(const json &j, const std::string &functionName) override; + json value(const CgNodePtr n) const override; }; struct CodeStatisticsHandler : public MetaDataHandler { const std::string toolname{"codeStatistics"}; - bool handles(const CgNodePtr n) const override { return false; } + bool handles(const CgNodePtr n) const override { return n->has(); } const std::string toolName() const override { return toolname; } void read(const json &j, const std::string &functionName) override; + json value(const CgNodePtr n) const override; }; struct NumConditionalBranchHandler : public MetaDataHandler { const std::string toolname{"numConditionalBranches"}; void read(const json &j, const std::string &functionName) override; - bool handles(const CgNodePtr n) const override { return false; } + bool handles(const CgNodePtr n) const override { return n->has(); } const std::string toolName() const override { return toolname; } + json value(const CgNodePtr n) const override; }; struct NumOperationsHandler : public MetaDataHandler { const std::string toolname{"numOperations"}; void read(const json &j, const std::string &functionName) override; - bool handles(const CgNodePtr n) const override { return false; } + bool handles(const CgNodePtr n) const override { return n->has(); } const std::string toolName() const override { return toolname; } + json value(const CgNodePtr n) const override; }; struct LoopDepthHandler : public MetaDataHandler { const std::string toolname{"loopDepth"}; void read(const json &j, const std::string &functionName) override; - bool handles(const CgNodePtr n) const override { return false; } + bool handles(const CgNodePtr n) const override { return n->has(); } const std::string toolName() const override { return toolname; } + json value(const CgNodePtr n) const override; }; struct GlobalLoopDepthHandler : public MetaDataHandler { const std::string toolname{"globalLoopDepth"}; void read(const json &j, const std::string &functionName) override; - bool handles(const CgNodePtr n) const override { return false; } + bool handles(const CgNodePtr n) const override { return n->has(); } const std::string toolName() const override { return toolname; } + json value(const CgNodePtr n) const override; }; // namespace retriever diff --git a/pgis/lib/include/loadImbalance/LIRetriever.h b/pgis/lib/include/loadImbalance/LIRetriever.h index af44f18..7d2483f 100644 --- a/pgis/lib/include/loadImbalance/LIRetriever.h +++ b/pgis/lib/include/loadImbalance/LIRetriever.h @@ -28,9 +28,10 @@ struct LIRetriever { class LoadImbalanceMetaDataHandler : public metacg::io::retriever::MetaDataHandler { public: - bool handles(const CgNodePtr n) const { return false; }; + bool handles(const CgNodePtr n) const { return n->has(); }; const std::string toolName() const override { return toolname; } void read(const nlohmann::json &j, const std::string &functionName) override; + nlohmann::json value(const CgNodePtr n) const override; private: const std::string toolname{"LIData"}; diff --git a/pgis/lib/src/libIPCG/MetaDataHandler.cpp b/pgis/lib/src/libIPCG/MetaDataHandler.cpp index 593ce6a..9c43131 100644 --- a/pgis/lib/src/libIPCG/MetaDataHandler.cpp +++ b/pgis/lib/src/libIPCG/MetaDataHandler.cpp @@ -8,8 +8,67 @@ namespace metacg { namespace io { + namespace retriever { +inline void to_json(json &j, const pira::BaseProfileData &data) { + j = nlohmann::json{{"numCalls", data.getNumberOfCalls()}, + {"timeInSeconds", data.getRuntimeInSeconds()}, + {"inclusiveRtInSeconds", data.getInclusiveRuntimeInSeconds()}}; +} + +/** + * This is the nlohmann::json way to serialize data and has been here before the MCGWriter existed. + * @param j + * @param data + */ +inline void to_json(json &j, const pira::PiraTwoData &data) { + auto &gOpts = ::pgis::config::GlobalConfig::get(); + auto rtOnly = gOpts.getAs("runtime-only"); + + auto rtAndParams = valTup(data.getRuntimeVec(), data.getExtrapParameters(), data.getNumReps()); + json experiments; + for (auto elem : rtAndParams) { + json exp{}; + exp["runtime"] = elem.first; + exp[elem.second.first] = elem.second.second; + experiments += exp; + } + if (!rtOnly) { + j = json{{"experiments", experiments}, + {"model", data.getExtrapModel()->getAsString(data.getExtrapModelConnector().getParamList())}}; + } else { + j = json{{"experiments", experiments}}; + } + spdlog::get("console")->debug("PiraTwoData to_json:\n{}", j.dump()); +} + +json BaseProfileDataHandler::value(const CgNodePtr n) const { + json j; + to_json(j, *(n->get())); + return j; +} + +void BaseProfileDataHandler::read([[maybe_unused]] const json &j, const std::string &functionName) { + spdlog::get("console")->trace("Running BaseProfileDataHandler::read"); + auto jIt = j[toolname]; + if (jIt.is_null()) { + spdlog::get("console")->trace("Could not retrieve metadata for {} in function {}", toolname, functionName); + return; + } + + auto numCalls = jIt["numCalls"].get(); + auto rtInSeconds = jIt["timeInSeconds"].get(); + auto inclRtInSeconds = jIt["inclusiveRtInSeconds"].get(); + auto node = mcgm->findOrCreateNode(functionName); + auto [has, obj] = node->checkAndGet(); + if (has) { + obj->setNumberOfCalls(numCalls); + obj->setRuntimeInSeconds(rtInSeconds); + obj->setInclusiveRuntimeInSeconds(inclRtInSeconds); + } +} + void PiraOneDataRetriever::read([[maybe_unused]] const json &j, const std::string &functionName) { spdlog::get("console")->trace("Running PiraOneMetaDataRetriever::read from json"); auto jIt = j[toolname]; @@ -45,7 +104,12 @@ bool PiraTwoDataRetriever::handles(const CgNodePtr n) const { return false; } -pira::PiraTwoData PiraTwoDataRetriever::value(const CgNodePtr n) const { return *(n->get()); } +json PiraTwoDataRetriever::value(const CgNodePtr n) const { + nlohmann::json j; + to_json(j, *(n->get())); + + return j; +} void PiraTwoDataRetriever::read([[maybe_unused]] const json &j, const std::string &functionName) { spdlog::get("console")->trace("Running PiraTwoDataRetriever::read from json"); @@ -65,6 +129,14 @@ void FilePropertyHandler::read(const json &j, const std::string &functionName) { node->addMetaData(md); } +json FilePropertyHandler::value(const CgNodePtr n) const { + json j; + auto fpData = n->get(); + j["origin"] = fpData->origin; + j["systemInclude"] = fpData->fromSystemInclude; + return j; +} + void CodeStatisticsHandler::read(const json &j, const std::string &functionName) { auto jIt = j[toolname]; if (jIt.is_null()) { @@ -77,6 +149,13 @@ void CodeStatisticsHandler::read(const json &j, const std::string &functionName) node->addMetaData(md); } +json CodeStatisticsHandler::value(const CgNodePtr n) const { + json j; + auto csData = n->get(); + j["numVars"] = csData->numVars; + return j; +} + void NumOperationsHandler::read(const json &j, const std::string &functionName) { auto jIt = j[toolname]; if (jIt.is_null()) { @@ -94,6 +173,17 @@ void NumOperationsHandler::read(const json &j, const std::string &functionName) md->numberOfMemoryAccesses = numberOfMemoryAccesses; node->addMetaData(md); } + +json NumOperationsHandler::value(const CgNodePtr n) const { + json j; + auto noData = n->get(); + j["numberOfIntOps"] = noData->numberOfIntOps; + j["numberOfFloatOps"] = noData->numberOfFloatOps; + j["numberOfControlFlowOps"] = noData->numberOfControlFlowOps; + j["numberOfMemoryAccesses"] = noData->numberOfMemoryAccesses; + return j; +} + void NumConditionalBranchHandler::read(const json &j, const std::string &functionName) { auto jIt = j[toolname]; if (jIt.is_null()) { @@ -106,6 +196,13 @@ void NumConditionalBranchHandler::read(const json &j, const std::string &functio node->addMetaData(md); } +json NumConditionalBranchHandler::value(const CgNodePtr n) const { + json j; + auto ncData = n->get(); + j["numConditionalBranches"] = ncData->numConditionalBranches; + return j; +} + void LoopDepthHandler::read(const json &j, const std::string &functionName) { auto jIt = j[toolname]; if (jIt.is_null()) { @@ -117,6 +214,14 @@ void LoopDepthHandler::read(const json &j, const std::string &functionName) { md->loopDepth = loopDepth; node->addMetaData(md); } + +json LoopDepthHandler::value(const CgNodePtr n) const { + json j; + auto ldData = n->get(); + j["loopDepth"] = ldData->loopDepth; + return j; +} + void GlobalLoopDepthHandler::read(const json &j, const std::string &functionName) { auto jIt = j[toolname]; if (jIt.is_null()) { @@ -128,6 +233,13 @@ void GlobalLoopDepthHandler::read(const json &j, const std::string &functionName md->globalLoopDepth = globalLoopDepth; node->addMetaData(md); } + +json GlobalLoopDepthHandler::value(const CgNodePtr n) const { + json j; + auto gldData = n->get(); + j["globalLoopDepth"] = gldData->globalLoopDepth; + return j; +} } // namespace retriever } // namespace io } // namespace metacg diff --git a/pgis/lib/src/loadImbalance/LIRetriever.cpp b/pgis/lib/src/loadImbalance/LIRetriever.cpp index b0f4214..04458cc 100644 --- a/pgis/lib/src/loadImbalance/LIRetriever.cpp +++ b/pgis/lib/src/loadImbalance/LIRetriever.cpp @@ -52,3 +52,9 @@ void LoadImbalanceMetaDataHandler::read(const nlohmann::json &j, const std::stri node->addMetaData(md); } } + +nlohmann::json LoadImbalanceMetaDataHandler::value(const CgNodePtr n) const { + nlohmann::json j; + to_json(j, *(n->get())); + return j; +} diff --git a/pgis/test/unit/CMakeLists.txt b/pgis/test/unit/CMakeLists.txt index 22336a1..5dd8c09 100644 --- a/pgis/test/unit/CMakeLists.txt +++ b/pgis/test/unit/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable( loadImbalance/LIMetricTest.cpp MCGManagerTest.cpp MCGReaderTest.cpp + MCGWriterTest.cpp ) target_link_libraries(basictests gtest_main) diff --git a/pgis/test/unit/IPCGAnnotation.cpp b/pgis/test/unit/IPCGAnnotation.cpp index a7cc1af..432d534 100644 --- a/pgis/test/unit/IPCGAnnotation.cpp +++ b/pgis/test/unit/IPCGAnnotation.cpp @@ -7,7 +7,7 @@ #include "gtest/gtest.h" #include -#include "MCGReader.h" +#include "MCGWriter.h" using namespace pira; using namespace metacg; diff --git a/pgis/test/unit/MCGReaderTest.cpp b/pgis/test/unit/MCGReaderTest.cpp index f6e19e8..34a8291 100644 --- a/pgis/test/unit/MCGReaderTest.cpp +++ b/pgis/test/unit/MCGReaderTest.cpp @@ -22,7 +22,7 @@ struct TestHandler : public metacg::io::retriever::MetaDataHandler { const std::string toolName() const override { return "TestMetaHandler"; } void read([[maybe_unused]] const json &j, const std::string &functionName) override { i++; } bool handles(const CgNodePtr n) const override { return false; } - int value(const CgNodePtr n) const { return i; } + json value(const CgNodePtr n) const { json j; return j << i; } }; TEST(VersionOneMCGReaderTest, EmptyJSON) { @@ -350,7 +350,7 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesTwoMetaDataHandler) { const std::string toolName() const override { return "TestMetaHandlerOne"; } void read([[maybe_unused]] const json &j, const std::string &functionName) override { i++; } bool handles(const CgNodePtr n) const override { return false; } - int value(const CgNodePtr n) const { return i; } + json value(const CgNodePtr n) const { json j; return j << i; } }; auto &mcgm = metacg::graph::MCGManager::get(); diff --git a/pgis/test/unit/MCGWriterTest.cpp b/pgis/test/unit/MCGWriterTest.cpp new file mode 100644 index 0000000..5786adb --- /dev/null +++ b/pgis/test/unit/MCGWriterTest.cpp @@ -0,0 +1,192 @@ +/** + * File: MCGWriterTest.cpp + * License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at + * https://github.com/tudasc/metacg/LICENSE.txt + */ +#include "gtest/gtest.h" + +#include "LoggerUtil.h" +#include "MCGManager.h" +#include "MCGWriter.h" +#include "MetaDataHandler.h" +#include "loadImbalance/LIRetriever.h" +#include "nlohmann/json.hpp" + +using namespace metacg; +using json = nlohmann::json; + +/** + * MetaDataHandler used for testing + */ +struct TestHandler : public metacg::io::retriever::MetaDataHandler { + int i{0}; + const std::string toolName() const override { return "TestMetaHandler"; } + void read([[maybe_unused]] const json &j, const std::string &functionName) override { i++; } + bool handles(const CgNodePtr n) const override { return false; } + json value(const CgNodePtr n) const { + json j; + return j << i; + } +}; + +namespace JsonFieldNames { +static const std::string cg{"_CG"}; +static const std::string md{"meta"}; +static const std::string bpd{"baseProfileData"}; +static const std::string pod{"numStatements"}; +static const std::string lid{"LIData"}; +} // namespace JsonFieldNames + +TEST(MCGWriterTest, EmptyGraph) { + json j; + loggerutil::getLogger(); + + auto &mcgm = metacg::graph::MCGManager::get(); + mcgm.reset(); + mcgm.addMetaHandler(); + mcgm.addMetaHandler(); + mcgm.addMetaHandler(); + + metacg::io::JsonSink jsSink; + metacg::io::MCGWriter mcgw(mcgm); + mcgw.write(jsSink); + + ASSERT_TRUE(j[JsonFieldNames::cg].is_null()); +} + +TEST(MCGWriterTest, OneNodeGraphNoHandlerAttached) { + json j; + loggerutil::getLogger(); + + auto &mcgm = metacg::graph::MCGManager::get(); + mcgm.reset(); + auto mainNode = mcgm.findOrCreateNode("main"); + + metacg::io::JsonSink jsSink; + metacg::io::MCGWriter mcgw(mcgm); + mcgw.write(jsSink); + + const auto js = jsSink.getJson(); + ASSERT_FALSE(js[JsonFieldNames::cg].is_null()); + + auto bpData = js[JsonFieldNames::cg]["main"][JsonFieldNames::md]; + for (auto it = bpData.begin(); it != bpData.end(); ++it) { + ASSERT_NE(it.key(), JsonFieldNames::bpd); + ASSERT_NE(it.key(), "numStatements"); + ASSERT_NE(it.key(), "LIData"); + } +} + +TEST(MCGWriterTest, OneNodeGraph) { + json j; + loggerutil::getLogger(); + + auto &mcgm = metacg::graph::MCGManager::get(); + mcgm.reset(); + auto mainNode = mcgm.findOrCreateNode("main"); + mcgm.addMetaHandler(); + mcgm.addMetaHandler(); + mcgm.addMetaHandler(); + + // How to use the MCGWriter + metacg::io::JsonSink jsSink; + metacg::io::MCGWriter mcgw(mcgm); + mcgw.write(jsSink); + + // Test that the fields are present in the generated json + const auto js = jsSink.getJson(); + ASSERT_FALSE(js[JsonFieldNames::cg].is_null()); + + // BaseProfileData + auto bpData = js[JsonFieldNames::cg]["main"][JsonFieldNames::md][JsonFieldNames::bpd]; + ASSERT_FALSE(bpData["numCalls"].is_null()); + ASSERT_EQ(bpData["numCalls"].get(), 0); + ASSERT_EQ(bpData["timeInSeconds"].get(), .0); + ASSERT_EQ(bpData["inclusiveRtInSeconds"].get(), .0); + + // PiraOneData, Fixme: the toolname of PIRA one data is old + auto poData = js[JsonFieldNames::cg]["main"][JsonFieldNames::md][JsonFieldNames::pod]; + ASSERT_FALSE(poData["numStatements"].is_null()); + ASSERT_EQ(poData["numStatements"].get(), 0); + + // LoadImbalanceDetection data + auto liData = js[JsonFieldNames::cg]["main"][JsonFieldNames::md][JsonFieldNames::lid]; + ASSERT_FALSE(liData.is_null()); + ASSERT_FALSE(liData["irrelevant"].is_null()); + ASSERT_FALSE(liData["visited"].is_null()); + ASSERT_EQ(liData["irrelevant"].get(), false); + ASSERT_EQ(liData["visited"].get(), false); +} + +TEST(MCGWriterTest, OneNodeGraphWithData) { + json j; + loggerutil::getLogger(); + + auto &mcgm = metacg::graph::MCGManager::get(); + mcgm.reset(); + auto mainNode = mcgm.findOrCreateNode("main"); + mcgm.addMetaHandler(); + mcgm.addMetaHandler(); + mcgm.addMetaHandler(); + + { + auto bpd = mainNode->get(); + bpd->setNumberOfCalls(42); + bpd->setRuntimeInSeconds(21); + bpd->setInclusiveRuntimeInSeconds(.666); + auto pod = mainNode->get(); + pod->setNumberOfStatements(64); + } + + // How to use the MCGWriter + metacg::io::JsonSink jsSink; + metacg::io::MCGWriter mcgw(mcgm); + mcgw.write(jsSink); + + // Test that the fields are present in the generated json + const auto js = jsSink.getJson(); + ASSERT_FALSE(js[JsonFieldNames::cg].is_null()); + + // BaseProfileData + auto bpData = js[JsonFieldNames::cg]["main"][JsonFieldNames::md][JsonFieldNames::bpd]; + ASSERT_FALSE(bpData["numCalls"].is_null()); + ASSERT_EQ(bpData["numCalls"].get(), 42); + ASSERT_EQ(bpData["timeInSeconds"].get(), 21); + ASSERT_EQ(bpData["inclusiveRtInSeconds"].get(), .666); + + // PiraOneData, Fixme: the toolname of PIRA one data is old + auto poData = js[JsonFieldNames::cg]["main"][JsonFieldNames::md][JsonFieldNames::pod]; + ASSERT_FALSE(poData["numStatements"].is_null()); + ASSERT_EQ(poData["numStatements"].get(), 64); + + // LoadImbalanceDetection data + auto liData = js[JsonFieldNames::cg]["main"][JsonFieldNames::md][JsonFieldNames::lid]; + ASSERT_FALSE(liData.is_null()); + ASSERT_FALSE(liData["irrelevant"].is_null()); + ASSERT_FALSE(liData["visited"].is_null()); + ASSERT_EQ(liData["irrelevant"].get(), false); + ASSERT_EQ(liData["visited"].get(), false); +} + +TEST(MCGWriterTest, OneNodeGraphNoDataForHandler) { + json j; + loggerutil::getLogger(); + + auto &mcgm = metacg::graph::MCGManager::get(); + mcgm.reset(); + auto mainNode = mcgm.findOrCreateNode("main"); + mcgm.addMetaHandler(); + + // How to use the MCGWriter + metacg::io::JsonSink jsSink; + metacg::io::MCGWriter mcgw(mcgm); + mcgw.write(jsSink); + + // Test that the fields are present in the generated json + const auto js = jsSink.getJson(); + ASSERT_FALSE(js[JsonFieldNames::cg].is_null()); + + // Global Loop Depth not attached, no meta data + auto metaData = js[JsonFieldNames::cg]["main"][JsonFieldNames::md]; + ASSERT_TRUE(metaData.is_null()); +} \ No newline at end of file diff --git a/pgis/test/unit/loadImbalance/LIMetaDataTest.cpp b/pgis/test/unit/loadImbalance/LIMetaDataTest.cpp index 7623682..5c09c37 100644 --- a/pgis/test/unit/loadImbalance/LIMetaDataTest.cpp +++ b/pgis/test/unit/loadImbalance/LIMetaDataTest.cpp @@ -6,7 +6,7 @@ #include "../../../../graph/include/Callgraph.h" #include "gtest/gtest.h" -#include +#include #include using namespace metacg; diff --git a/pgis/tool/PGISMain.cpp b/pgis/tool/PGISMain.cpp index 91372c3..4aa9a20 100644 --- a/pgis/tool/PGISMain.cpp +++ b/pgis/tool/PGISMain.cpp @@ -10,6 +10,7 @@ #include "CubeReader.h" #include "DotReader.h" #include "MCGReader.h" +#include "MCGWriter.h" #include "ExtrapEstimatorPhase.h" #include "IPCGEstimatorPhase.h" @@ -357,7 +358,7 @@ int main(int argc, char **argv) { if (!pConfig.getLIConfig()) { spdlog::get("errconsole") ->error("Provide configuration for load imbalance detection. Refer to PIRA's README for further details."); - return (EXIT_FAILURE); + return EXIT_FAILURE; } cg.registerEstimatorPhase( new LoadImbalance::LIEstimatorPhase(std::move(pConfig.getLIConfig()))); // attention: moves out liConfig! @@ -387,7 +388,7 @@ int main(int argc, char **argv) { auto &pConfig = pgis::config::ParameterConfig::get(); if (!pConfig.getPiraIIConfig()) { console->error("Provide PIRA II configuration in order to use Extra-P estimators."); - return (EXIT_FAILURE); + return EXIT_FAILURE; } cg.attachExtrapModels(); @@ -419,5 +420,14 @@ int main(int argc, char **argv) { cg.applyRegisteredPhases(); } + // Example use of MetaCG writer + { + metacg::io::JsonSink jsSink; + metacg::io::MCGWriter mcgw(mcgm); + mcgw.write(jsSink); + std::ofstream ofile("filename"); + jsSink.output(ofile); + } + return EXIT_SUCCESS; } From 073cf190732180686e9d5bf33901d754b7681c12 Mon Sep 17 00:00:00 2001 From: jr Date: Fri, 25 Mar 2022 12:17:32 +0100 Subject: [PATCH 07/12] Fix build error caused by ambiguous operator/static assert --- pgis/lib/include/libIPCG/MetaDataHandler.h | 3 ++- pgis/test/unit/MCGReaderTest.cpp | 12 ++++++++++-- pgis/test/unit/MCGWriterTest.cpp | 3 ++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pgis/lib/include/libIPCG/MetaDataHandler.h b/pgis/lib/include/libIPCG/MetaDataHandler.h index 5bdfc2b..0b1ca90 100644 --- a/pgis/lib/include/libIPCG/MetaDataHandler.h +++ b/pgis/lib/include/libIPCG/MetaDataHandler.h @@ -43,7 +43,8 @@ struct TestHandler : public MetaDataHandler { bool handles(const CgNodePtr n) const override { return false; } json value(const CgNodePtr n) const override { json j; - return j << i; + j = i; + return j; } }; diff --git a/pgis/test/unit/MCGReaderTest.cpp b/pgis/test/unit/MCGReaderTest.cpp index 34a8291..3a59bfc 100644 --- a/pgis/test/unit/MCGReaderTest.cpp +++ b/pgis/test/unit/MCGReaderTest.cpp @@ -22,7 +22,11 @@ struct TestHandler : public metacg::io::retriever::MetaDataHandler { const std::string toolName() const override { return "TestMetaHandler"; } void read([[maybe_unused]] const json &j, const std::string &functionName) override { i++; } bool handles(const CgNodePtr n) const override { return false; } - json value(const CgNodePtr n) const { json j; return j << i; } + json value(const CgNodePtr n) const { + json j; + j = i; + return j; + } }; TEST(VersionOneMCGReaderTest, EmptyJSON) { @@ -350,7 +354,11 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesTwoMetaDataHandler) { const std::string toolName() const override { return "TestMetaHandlerOne"; } void read([[maybe_unused]] const json &j, const std::string &functionName) override { i++; } bool handles(const CgNodePtr n) const override { return false; } - json value(const CgNodePtr n) const { json j; return j << i; } + json value(const CgNodePtr n) const { + json j; + j = 1; + return j; + } }; auto &mcgm = metacg::graph::MCGManager::get(); diff --git a/pgis/test/unit/MCGWriterTest.cpp b/pgis/test/unit/MCGWriterTest.cpp index 5786adb..5887565 100644 --- a/pgis/test/unit/MCGWriterTest.cpp +++ b/pgis/test/unit/MCGWriterTest.cpp @@ -25,7 +25,8 @@ struct TestHandler : public metacg::io::retriever::MetaDataHandler { bool handles(const CgNodePtr n) const override { return false; } json value(const CgNodePtr n) const { json j; - return j << i; + j = i; + return j; } }; From 7a147589f94f382f51535eee734435294f48ab51 Mon Sep 17 00:00:00 2001 From: JP Lehr Date: Mon, 28 Mar 2022 23:32:14 +0200 Subject: [PATCH 08/12] Updates version for next release and fixes CONTRIBUTING info --- CMakeLists.txt | 2 +- CONTRIBUTING.md | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 90f2f94..46685cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ endif() # End of require out-of-source builds # Project information -set(METACG_VERSION 0.3.0) +set(METACG_VERSION 0.4.0) project( MetaCG VERSION ${METACG_VERSION} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e3f3fde..0bed777 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,25 +6,24 @@ In case you want to contribute, please reach out through the issues and we see h ## Issues -We are aware that metacg has open issues, i.e., a correct handling of function pointers, etc. +We are aware that MetaCG has open issues, i.e., a correct handling of function pointers, etc. If you find a new issue that is not already reported in github, please feel free to open an issue. The issue should state clearly what is the problem, and what you did to get there. -If you can, please provide a MWE in the issue, and the tool output. -Please also share, what you have the tool expected to do. +If you can, please provide a minimal working example in the issue, and the tool output. +Please also share, what you expected the tool to do. ## Workflow -Please contribute only via pull requests and (a) keep your commit history clean, or (b) it will be squashed during merge. +We currently accept changes via merge requests in our non-public repository. +All history will be squashed in the merge. -**Internal**: Please branch from `master` for fixes and from `devel` for researchy new stuff. -`master` is automatically mirrored to github, so, please, only merge to `master`, if the CI builds succeed. -Preceed a branch either with `fix/` or with `feat/` to indicate it. +**Internal**: Please always branch from `devel`, and only in very particular bugfix cases (really critical ones) ask in our slack what to do. +Preceed a branch either with `fix/` or with `feat/` to indicate what it is meant to do, followed by the issue number. ## Tests Every contribution must add reasonable tests. -(We use a Gitlab internally for development, so the github CI / actions are not yet set up.) ## Commit messages From 982c2ac4219c7538dc76c2e2ec47e8ce12482e81 Mon Sep 17 00:00:00 2001 From: "Heldmann, Tim" Date: Fri, 1 Apr 2022 12:23:10 +0200 Subject: [PATCH 09/12] Extends MCGManager to manage multiple call graphs at once. --- graph/include/MCGManager.h | 196 +++++++++++++++--- graph/src/MCGReader.cpp | 5 +- graph/src/MCGWriter.cpp | 2 +- pgis/lib/include/CallgraphManager.h | 2 +- pgis/test/unit/IPCGEstimatorPhaseTest.cpp | 47 +++-- pgis/test/unit/MCGManagerTest.cpp | 12 +- pgis/test/unit/MCGReaderTest.cpp | 44 ++-- pgis/test/unit/MCGWriterTest.cpp | 10 +- .../loadImbalance/LIEstimatorPhaseTest.cpp | 42 ++-- pgis/tool/PGISMain.cpp | 6 +- pgis/tool/StmtPrinterMain.cpp | 5 +- 11 files changed, 267 insertions(+), 104 deletions(-) diff --git a/graph/include/MCGManager.h b/graph/include/MCGManager.h index 2a6b2dc..e4dee0a 100644 --- a/graph/include/MCGManager.h +++ b/graph/include/MCGManager.h @@ -19,17 +19,42 @@ namespace graph { */ class MCGManager { public: - static MCGManager &get() { static MCGManager instance; return instance; } - void reset() { - graph.clear(); + /** + * Resets the the whole manager + */ + void resetManager() { metaHandlers.clear(); + managedGraphs.clear(); + activeGraph = nullptr; } + /** + * Resets the current active graph to be empty + */ + bool resetActiveGraph() { + if (activeGraph) { + activeGraph->clear(); + // just because a graph exists, doesn't mean it has attached metadata + // I think this is the best way to do it + // until map::contains comes with c++20 + // I expect most callgraphs to have attached metadata + // so the exception should be rare + try { + metaHandlers.at(activeGraph).clear(); + } catch (const std::out_of_range &ex) { + } + return true; + } else { + std::cerr << "Graph manager could not reset active Graph, no active graph exists\n"; + assert(false&&"Graph manager could not reset active Graph, no active graph exists"); + return false; + } + } /** * Registers the MetaDataHandler and takes ownership of the object. * @tparam T @@ -37,9 +62,17 @@ class MCGManager { * @param args */ template - void addMetaHandler(Args... args) { - metaHandlers.emplace_back(std::make_unique(args...)); - metaHandlers.back()->registerMCGManager(this); + bool addMetaHandler(Args... args) { + if (activeGraph != nullptr) { + metaHandlers[activeGraph].emplace_back(std::make_unique(args...)); + metaHandlers[activeGraph].back()->registerMCGManager(this); + return true; + } else { + auto errconsole = spdlog::get("errconsole"); + errconsole->error("Graph manager could not add metadata handler, no active graph exists"); + assert(false&&"Graph manager could not add metadata handler, no active graph exists"); + return false; + } } /** @@ -48,8 +81,12 @@ class MCGManager { */ std::vector getMetaHandlers() const { std::vector handler; - for (const auto &mh : metaHandlers) { - handler.push_back(mh.get()); + // the tests expect an empty vector of metadata handlers to be returned if the active graph is either not metadata + // annotated or doesn't exist + if (metaHandlers.count(activeGraph) != 0) { + for (const auto &mh : metaHandlers.at(activeGraph)) { + handler.push_back(mh.get()); + } } return handler; } @@ -59,40 +96,131 @@ class MCGManager { * @param parentName function name of calling function * @param childName function name of called function */ - //Note: this is kept for compatibility reasons - // remove this once transition is complete - void addEdge(const std::string &parentName, const std::string &childName) { - graph.addEdge(parentName, childName); - }; + // Note: this is kept for compatibility reasons + // remove this once transition is complete + bool addEdge(const std::string &parentName, const std::string &childName) { + if (activeGraph != nullptr) { + activeGraph->addEdge(parentName, childName); + return true; + } else { + auto errconsole = spdlog::get("errconsole"); + errconsole->error("Graph manager could not create edge between %s and %s, no active graph exists",parentName,childName ); + assert(false&&"Graph manager could not create edge between given nodes, no active graph exists"); + return false; + } + } /** * Inserts an edge from parentNode to childNode * @param parentNode function node of calling function * @param childNode function node of called function */ - //Note: this is kept for compatibility reasons - // remove this once transition is complete - void addEdge(CgNodePtr parentNode, CgNodePtr childNode) { - graph.addEdge(parentNode, childNode); + // Note: this is kept for compatibility reasons + // remove this once transition is complete + bool addEdge(CgNodePtr parentNode, CgNodePtr childNode) { + if (activeGraph != nullptr) { + activeGraph->addEdge(parentNode, childNode); + return true; + } else { + auto errconsole = spdlog::get("errconsole"); + errconsole->error("Graph manager could not create edge, no active graph exists\n"); + assert(false&&"Graph manager could not create edge, no active graph exists"); + return false; + } } - /** - * Returns the node for @param name - * If no node exists yet, it creates a new one. + * Returns the node for given identifier, if no node exists yet, it creates a new one. + * @param name name to identify the node by + * @returns a shared pointer to the requested node */ - //Note: this is kept for compatibility reasons - // remove this once transition is complete - CgNodePtr findOrCreateNode(const std::string &name) { - return graph.getOrInsertNode(name); - } + + // Note: this is kept for compatibility reasons + // remove this once transition is complete + CgNodePtr findOrCreateNode(const std::string &name) { return activeGraph->getOrInsertNode(name); } // Delegates to the underlying graph - metacg::Callgraph::ContainerT::iterator begin() const { return graph.begin(); } - metacg::Callgraph::ContainerT::iterator end() const { return graph.end(); } - metacg::Callgraph::ContainerT::const_iterator cbegin() const { return graph.cbegin(); } - metacg::Callgraph::ContainerT::const_iterator cend() const { return graph.cend(); } - size_t size() { return graph.size(); } - [[nodiscard]] metacg::Callgraph &getCallgraph() { return graph; } + metacg::Callgraph::ContainerT::iterator begin() { return activeGraph->begin(); } + metacg::Callgraph::ContainerT::iterator end() { return activeGraph->end(); } + metacg::Callgraph::ContainerT::const_iterator cbegin() const { return activeGraph->cbegin(); } + metacg::Callgraph::ContainerT::const_iterator cend() const { return activeGraph->cend(); } + size_t size() { return activeGraph->size(); } + + // Delegates iterators for all managed graphs + std::unordered_map>::iterator graphs_begin() { + return managedGraphs.begin(); + } + std::unordered_map>::iterator graphs_end() { + return managedGraphs.end(); + } + std::unordered_map>::const_iterator graphs_cbegin() const { + return managedGraphs.cbegin(); + } + std::unordered_map>::const_iterator graphs_cend() const { + return managedGraphs.cend(); + } + + size_t graphs_size() { return managedGraphs.size(); } + + [[nodiscard]] metacg::Callgraph *getCallgraph() { return activeGraph; } + [[nodiscard]] metacg::Callgraph *getOrCreateCallgraph(const std::string &name) { return managedGraphs[name].get(); } + + /** + * Sets a callgraph to be the currently active one + * @param callgraph the name the callgraph is identified by + * @returns true if the callgraph could be set active,\n false if it is not part of the managed callgraphs + * + **/ + // Todo: write a test for this + bool setActive(const std::string &callgraph) { + // I think this is the best way to do it + // until map::contains comes with c++20 + // another implementation would be to + // generate a new graph if no matching graph was found + // I expect most users to only set a graph active + // once it has been added, so the exception should be rare + try { + activeGraph = managedGraphs.at(callgraph).get(); + } catch (const std::out_of_range &ex) { + auto errconsole = spdlog::get("errconsole"); + errconsole->error("Could not set graph %s to active, graph does not exist",callgraph); + assert(false&&"Could not set graph to active, graph does not exist"); + return false; + } + return true; + } + + /** + * Checks if a callgraph is the currently active one + * @param callgraph the name the callgraph is identified by + * @returns true if the callgraph is the active one \n false if the callgraph is inactive or doesn't exist + **/ + // TODO: write a test for this + bool assertActive(const std::string &callgraph) { + try { + return managedGraphs.at(callgraph).get() == activeGraph; + } catch (const std::out_of_range &ex) { + auto errconsole = spdlog::get("errconsole"); + errconsole->error("Graph: %s is not part of managed graphs",callgraph); + assert(false&&"Graph is not part of managed graphs"); + return false; + } + } + + /** + * Transfers ownership of a graph to the graph manager + * @param name the name to identify the graph with + * @param callgraph the unique pointer to the callgraph + * @param setActive optional flag to set the newly added graph as the active one + **/ + // Todo: write a test for this + bool addToManagedGraphs(std::string name, std::unique_ptr callgraph, bool setActive = true) { + assert(callgraph.get()!= nullptr&&"Could not add to managed graphs, given graph was null"); + managedGraphs[name] = std::move(callgraph); + if (setActive) { + activeGraph = managedGraphs[name].get(); + } + return true; + } ~MCGManager() = default; @@ -104,8 +232,10 @@ class MCGManager { MCGManager &operator=(const MCGManager &other) = delete; MCGManager &operator=(MCGManager &&other) = delete; - metacg::Callgraph graph; - std::vector> metaHandlers{}; + std::unordered_map> managedGraphs; + metacg::Callgraph *activeGraph = nullptr; + std::unordered_map>> + metaHandlers{}; }; } // namespace graph } // namespace metacg diff --git a/graph/src/MCGReader.cpp b/graph/src/MCGReader.cpp index b59da1e..733f1a4 100644 --- a/graph/src/MCGReader.cpp +++ b/graph/src/MCGReader.cpp @@ -169,8 +169,7 @@ void VersionOneMetaCGReader::read(metacg::graph::MCGManager &cgManager) { // set load imbalance flags in CgNode for (const auto pfi : functions) { - std::optional opt_f = cgManager.getCallgraph().getNode(pfi.first); - + std::optional opt_f = cgManager.getCallgraph()->getNode(pfi.first); if (opt_f.has_value()) { CgNodePtr node = opt_f.value(); node->get()->setVirtual(pfi.second.isVirtual); @@ -189,7 +188,7 @@ void VersionOneMetaCGReader::read(metacg::graph::MCGManager &cgManager) { void VersionOneMetaCGReader::addNumStmts(metacg::graph::MCGManager &cgm) { for (const auto &[k, fi] : functions) { auto g = cgm.getCallgraph(); - auto node = g.getNode(fi.functionName); + auto node = g->getNode(fi.functionName); assert(node != nullptr && "Nodes with #statements attached should be available"); if (node->has()){ auto pod = node->get(); diff --git a/graph/src/MCGWriter.cpp b/graph/src/MCGWriter.cpp index 8c32835..53d39b9 100644 --- a/graph/src/MCGWriter.cpp +++ b/graph/src/MCGWriter.cpp @@ -37,7 +37,7 @@ void metacg::io::MCGWriter::write(JsonSink &js) { nlohmann::json j; attachFormatTwoHeader(j); - for (const auto &n : mcgManager.getCallgraph()) { + for (const auto &n : *mcgManager.getCallgraph()) { createNodeData(n, j); // general node data? createAndAddMetaData(n, mcgManager, j); } diff --git a/pgis/lib/include/CallgraphManager.h b/pgis/lib/include/CallgraphManager.h index b91f7f3..ea3f604 100644 --- a/pgis/lib/include/CallgraphManager.h +++ b/pgis/lib/include/CallgraphManager.h @@ -98,7 +98,7 @@ class PiraMCGProcessor { void attachExtrapModels(); - void setCG(Callgraph &newGraph) { graph = newGraph; } + void setCG(Callgraph& newGraph) {graph = newGraph; } private: // this set represents the call graph during the actual computation diff --git a/pgis/test/unit/IPCGEstimatorPhaseTest.cpp b/pgis/test/unit/IPCGEstimatorPhaseTest.cpp index 5da4d28..c014603 100644 --- a/pgis/test/unit/IPCGEstimatorPhaseTest.cpp +++ b/pgis/test/unit/IPCGEstimatorPhaseTest.cpp @@ -17,18 +17,20 @@ class IPCGEstimatorPhaseBasic : public ::testing::Test { void SetUp() override { loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); } }; TEST_F(IPCGEstimatorPhaseBasic, EmptyCG) { + // XXX this is not ideal, but required for the death assert LOGGERUTIL_ENABLE_ERRORS_LOCAL - Config cfg; auto &mcgm = metacg::graph::MCGManager::get(); auto &cm = metacg::pgis::PiraMCGProcessor::get(); - cm.setCG(mcgm.getCallgraph()); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); + cm.setCG(*mcgm.getCallgraph()); cm.setConfig(&cfg); cm.setNoOutput(); StatementCountEstimatorPhase sce(10); @@ -41,18 +43,19 @@ TEST_F(IPCGEstimatorPhaseBasic, OneNodeCG) { Config cfg; auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - // mcgm.reset(); + mcgm.resetManager(); cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); ASSERT_NE(mainNode, nullptr); - cm.setCG(mcgm.getCallgraph()); - ASSERT_FALSE(mcgm.getCallgraph().isEmpty()); + cm.setCG(*mcgm.getCallgraph()); + ASSERT_FALSE(mcgm.getCallgraph()->isEmpty()); StatementCountEstimatorPhase sce(10); sce.setNoReport(); cm.registerEstimatorPhase(&sce); cm.applyRegisteredPhases(); - auto graph = mcgm.getCallgraph(); + auto graph = *mcgm.getCallgraph(); ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(0, sce.getNumStatements(mainNode)); cm.removeAllEstimatorPhases(); @@ -64,11 +67,12 @@ TEST_F(IPCGEstimatorPhaseBasic, TwoNodeCG) { auto &mcgm = metacg::graph::MCGManager::get(); cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); auto childNode = mcgm.findOrCreateNode("child1"); mcgm.addEdge(mainNode, childNode); // cm.putEdge("main", "main.c", 1, "child1", 100, 1.2, 0, 0); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); StatementCountEstimatorPhase sce(10); sce.setNoReport(); cm.registerEstimatorPhase(&sce); @@ -85,6 +89,7 @@ TEST_F(IPCGEstimatorPhaseBasic, OneNodeCGwStmt) { auto &mcgm = metacg::graph::MCGManager::get(); cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); const auto &[has, data] = mainNode->checkAndGet(); if (has) { @@ -93,7 +98,7 @@ TEST_F(IPCGEstimatorPhaseBasic, OneNodeCGwStmt) { } else { assert(false && "Nodes should have PIRA I data attached."); } - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); // cm.putNumberOfStatements("main", 12, true); StatementCountEstimatorPhase sce(10); sce.setNoReport(); @@ -112,6 +117,7 @@ TEST_F(IPCGEstimatorPhaseBasic, TwoNodeCGwStmt) { cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); pira::setPiraOneData(mainNode, 12, true); @@ -120,7 +126,7 @@ TEST_F(IPCGEstimatorPhaseBasic, TwoNodeCGwStmt) { childNode->setReachable(); mcgm.addEdge(mainNode, childNode); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); StatementCountEstimatorPhase sce(10); sce.setNoReport(); @@ -141,6 +147,7 @@ TEST_F(IPCGEstimatorPhaseBasic, ThreeNodeCGwStmt) { cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); pira::setPiraOneData(mainNode, 12, true); @@ -154,7 +161,7 @@ TEST_F(IPCGEstimatorPhaseBasic, ThreeNodeCGwStmt) { mcgm.addEdge(mainNode, childNode2); childNode2->setReachable(); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); StatementCountEstimatorPhase sce(10); sce.setNoReport(); @@ -175,6 +182,7 @@ TEST_F(IPCGEstimatorPhaseBasic, ThreeNodeCycleCGwStmt) { cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); pira::setPiraOneData(mainNode, 12, true); @@ -189,7 +197,7 @@ TEST_F(IPCGEstimatorPhaseBasic, ThreeNodeCycleCGwStmt) { mcgm.addEdge(childNode2, childNode); childNode2->setReachable(); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); StatementCountEstimatorPhase sce(10); sce.setNoReport(); @@ -211,6 +219,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FourNodeCGwStmt) { cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); pira::setPiraOneData(mainNode, 12, true); @@ -229,7 +238,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FourNodeCGwStmt) { mcgm.addEdge(childNode, childNode3); childNode3->setReachable(); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); StatementCountEstimatorPhase sce(10); sce.setNoReport(); @@ -251,6 +260,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FourNodeDiamondCGwStmt) { cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); pira::setPiraOneData(mainNode, 12, true); @@ -270,7 +280,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FourNodeDiamondCGwStmt) { mcgm.addEdge(childNode2, childNode3); childNode3->setReachable(); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); StatementCountEstimatorPhase sce(10); sce.setNoReport(); @@ -301,6 +311,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FiveNodeDiamondCGwStmt) { cm.setConfig(&cfg); cm.setNoOutput(); + mcgm.addToManagedGraphs("graph",std::make_unique()); auto mainNode = mcgm.findOrCreateNode("main"); pira::setPiraOneData(mainNode, 12, true); @@ -325,7 +336,7 @@ TEST_F(IPCGEstimatorPhaseBasic, FiveNodeDiamondCGwStmt) { mcgm.addEdge(childNode3, childNode4); childNode4->setReachable(); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); StatementCountEstimatorPhase sce(10); sce.setNoReport(); @@ -365,7 +376,9 @@ class IPCGEstimatorPhaseTest : public ::testing::Test { loggerutil::getLogger(); auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); + cm.setConfig(new Config()); cm.setNoOutput(); auto mainNode = mcgm.findOrCreateNode("main"); @@ -380,7 +393,7 @@ class IPCGEstimatorPhaseTest : public ::testing::Test { createCalleeNode("child6", "child5", 2, 0.8, 1000); createCalleeNode("child7", "child6", 1, 20.2, 1002); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); } }; diff --git a/pgis/test/unit/MCGManagerTest.cpp b/pgis/test/unit/MCGManagerTest.cpp index 23e86aa..b694fb1 100644 --- a/pgis/test/unit/MCGManagerTest.cpp +++ b/pgis/test/unit/MCGManagerTest.cpp @@ -18,14 +18,18 @@ class MCGManagerTest : public ::testing::Test { void SetUp() override { loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); } }; TEST_F(MCGManagerTest, EmptyCG) { + auto &mcgm = metacg::graph::MCGManager::get(); ASSERT_EQ(0, mcgm.size()); - auto graph = mcgm.getCallgraph(); + + auto graph = *mcgm.getCallgraph(); + ASSERT_TRUE(graph.isEmpty()); ASSERT_EQ(false, graph.hasNode("main")); ASSERT_EQ(nullptr, graph.getMain()); @@ -36,7 +40,7 @@ TEST_F(MCGManagerTest, OneNodeCG) { auto &mcgm = metacg::graph::MCGManager::get(); mcgm.findOrCreateNode("main"); auto nPtr = mcgm.findOrCreateNode("main"); - auto graph = mcgm.getCallgraph(); + auto graph = *mcgm.getCallgraph(); ASSERT_FALSE(graph.isEmpty()); ASSERT_NE(nullptr, graph.getMain()); ASSERT_EQ(nPtr, graph.getMain()); @@ -49,7 +53,7 @@ TEST_F(MCGManagerTest, TwoNodeCG) { mcgm.addEdge("main", "child1"); ASSERT_EQ(mainNode, mcgm.findOrCreateNode("main")); ASSERT_EQ(childNode, mcgm.findOrCreateNode("child1")); - auto graph = mcgm.getCallgraph(); + auto graph = *mcgm.getCallgraph(); ASSERT_EQ(mainNode, graph.getMain()); ASSERT_EQ(childNode, graph.getNode("child1")); } diff --git a/pgis/test/unit/MCGReaderTest.cpp b/pgis/test/unit/MCGReaderTest.cpp index 3a59bfc..47b055e 100644 --- a/pgis/test/unit/MCGReaderTest.cpp +++ b/pgis/test/unit/MCGReaderTest.cpp @@ -38,12 +38,13 @@ TEST(VersionOneMCGReaderTest, EmptyJSON) { // Config c; auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); metacg::io::JsonSource js(j); metacg::io::VersionOneMetaCGReader mcgReader(js); mcgReader.read(mcgm); - const Callgraph &graph = mcgm.getCallgraph(); + const Callgraph &graph = *mcgm.getCallgraph(); ASSERT_EQ(graph.size(), 0); } @@ -62,12 +63,13 @@ TEST(VersionOneMCGReaderTest, SimpleJSON) { // Config c; auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); metacg::io::JsonSource js(j); metacg::io::VersionOneMetaCGReader mcgReader(js); mcgReader.read(mcgm); - Callgraph &graph = mcgm.getCallgraph(); + Callgraph &graph = *mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 1); CgNodePtr mainNode = graph.getNode("main"); @@ -104,12 +106,13 @@ TEST(VersionOneMCGReaderTest, MultiNodeJSON) { // Config c; auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); metacg::io::JsonSource js(j); metacg::io::VersionOneMetaCGReader mcgReader(js); mcgReader.read(mcgm); - Callgraph &graph = mcgm.getCallgraph(); + Callgraph &graph = *mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 2); CgNodePtr mainNode = graph.getNode("main"); @@ -149,7 +152,8 @@ TEST(VersionTwoMetaCGReaderTest, EmptyJSON) { // No MetaData Reader added to CGManager auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); metacg::io::JsonSource js(j); metacg::io::VersionTwoMetaCGReader mcgReader(js); ASSERT_THROW(mcgReader.read(mcgm), std::runtime_error); @@ -168,7 +172,8 @@ TEST(VersionTwoMetaCGReaderTest, EmptyCG) { // No MetaData Reader added to CGManager auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); metacg::io::JsonSource js(j); metacg::io::VersionTwoMetaCGReader mcgReader(js); ASSERT_THROW(mcgReader.read(mcgm), std::runtime_error); @@ -183,7 +188,8 @@ TEST(VersionTwoMetaCGReaderTest, SingleMetaDataHandlerEmptyJSON) { // // Config c; auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); mcgm.addMetaHandler(); metacg::io::JsonSource js(j); @@ -211,12 +217,13 @@ TEST(VersionTwoMetaCGReaderTest, OneNodeNoMetaDataHandler) { // No MetaData Reader added to CGManager auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); metacg::io::JsonSource js(j); metacg::io::VersionTwoMetaCGReader mcgReader(js); mcgReader.read(mcgm); - auto graph = mcgm.getCallgraph(); + auto graph = *mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 1); const auto mainNode = graph.getMain(); @@ -255,12 +262,13 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesNoMetaDataHandler) { // No MetaData Reader added to CGManager auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); metacg::io::JsonSource js(j); metacg::io::VersionTwoMetaCGReader mcgReader(js); mcgReader.read(mcgm); - auto graph = mcgm.getCallgraph(); + auto graph = *mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 2); const auto mainNode = graph.getMain(); @@ -298,14 +306,15 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesOneMetaDataHandler) { // Config c; auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); mcgm.addMetaHandler(); metacg::io::JsonSource js(j); metacg::io::VersionTwoMetaCGReader mcgReader(js); mcgReader.read(mcgm); - auto graph = mcgm.getCallgraph(); + auto graph = *mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 2); const auto mainNode = graph.getMain(); @@ -362,7 +371,8 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesTwoMetaDataHandler) { }; auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); mcgm.addMetaHandler(); mcgm.addMetaHandler(); @@ -370,7 +380,7 @@ TEST(VersionTwoMetaCGReaderTest, TwoNodesTwoMetaDataHandler) { metacg::io::VersionTwoMetaCGReader mcgReader(js); mcgReader.read(mcgm); - auto graph = mcgm.getCallgraph(); + auto graph = *mcgm.getCallgraph(); EXPECT_EQ(graph.size(), 2); const auto mainNode = graph.getMain(); diff --git a/pgis/test/unit/MCGWriterTest.cpp b/pgis/test/unit/MCGWriterTest.cpp index 5887565..7f9b702 100644 --- a/pgis/test/unit/MCGWriterTest.cpp +++ b/pgis/test/unit/MCGWriterTest.cpp @@ -43,7 +43,7 @@ TEST(MCGWriterTest, EmptyGraph) { loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetActiveGraph(); mcgm.addMetaHandler(); mcgm.addMetaHandler(); mcgm.addMetaHandler(); @@ -60,7 +60,7 @@ TEST(MCGWriterTest, OneNodeGraphNoHandlerAttached) { loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetActiveGraph(); auto mainNode = mcgm.findOrCreateNode("main"); metacg::io::JsonSink jsSink; @@ -83,7 +83,7 @@ TEST(MCGWriterTest, OneNodeGraph) { loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetActiveGraph(); auto mainNode = mcgm.findOrCreateNode("main"); mcgm.addMetaHandler(); mcgm.addMetaHandler(); @@ -124,7 +124,7 @@ TEST(MCGWriterTest, OneNodeGraphWithData) { loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetActiveGraph(); auto mainNode = mcgm.findOrCreateNode("main"); mcgm.addMetaHandler(); mcgm.addMetaHandler(); @@ -174,7 +174,7 @@ TEST(MCGWriterTest, OneNodeGraphNoDataForHandler) { loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetActiveGraph(); auto mainNode = mcgm.findOrCreateNode("main"); mcgm.addMetaHandler(); diff --git a/pgis/test/unit/loadImbalance/LIEstimatorPhaseTest.cpp b/pgis/test/unit/loadImbalance/LIEstimatorPhaseTest.cpp index fc04b9d..23f629d 100644 --- a/pgis/test/unit/loadImbalance/LIEstimatorPhaseTest.cpp +++ b/pgis/test/unit/loadImbalance/LIEstimatorPhaseTest.cpp @@ -31,18 +31,19 @@ TEST_F(LIEstimatorPhaseTest, EmptyCG) { Config cfg; auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); cm.removeAllEstimatorPhases(); cm.setConfig(&cfg); cm.setNoOutput(); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); auto liConfig = std::make_unique( LoadImbalance::LIConfig{LoadImbalance::MetricType::Efficiency, 1.2, 0.1, LoadImbalance::ContextStrategy::None, 0, LoadImbalance::ChildRelevanceStrategy::ConstantThreshold, 5, 0.0}); LoadImbalance::LIEstimatorPhase lie(std::move(liConfig)); cm.registerEstimatorPhase(&lie); - ASSERT_TRUE(mcgm.getCallgraph().isEmpty()); + ASSERT_TRUE(mcgm.getCallgraph()->isEmpty()); ASSERT_TRUE(cm.getCallgraph(&cm).isEmpty()); ASSERT_DEATH(cm.applyRegisteredPhases(), "Running the processor on empty graph. Need to construct graph."); } @@ -52,7 +53,8 @@ TEST_F(LIEstimatorPhaseTest, AllCases) { auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); cm.removeAllEstimatorPhases(); cm.setConfig(&cfg); cm.setNoOutput(); @@ -116,11 +118,11 @@ TEST_F(LIEstimatorPhaseTest, AllCases) { childNode4->get()->setComesFromCube(); childNode5->get()->setComesFromCube(); - for (CgNodePtr n : mcgm.getCallgraph()) { + for (CgNodePtr n : *mcgm.getCallgraph()) { n->get()->setNumberOfStatements(100); } - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); // apply estimator phases auto liConfig = std::make_unique( @@ -152,9 +154,10 @@ TEST_F(LIEstimatorPhaseTest, Virtual) { Config cfg; auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); cm.removeAllEstimatorPhases(); - ASSERT_TRUE(mcgm.getCallgraph().isEmpty()); + ASSERT_TRUE(mcgm.getCallgraph()->isEmpty()); cm.setConfig(&cfg); cm.setNoOutput(); @@ -177,11 +180,11 @@ TEST_F(LIEstimatorPhaseTest, Virtual) { child->addChildNode(grandchild); grandchild->addChildNode(grandgrandchild); - for (CgNodePtr n : mcgm.getCallgraph()) { + for (CgNodePtr n : *mcgm.getCallgraph()) { n->get()->setNumberOfStatements(100); } - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); // apply estimator phases auto liConfig = std::make_unique( @@ -204,7 +207,8 @@ TEST_F(LIEstimatorPhaseTest, AllPathsToMain) { Config cfg; auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); cm.removeAllEstimatorPhases(); cm.setConfig(&cfg); cm.setNoOutput(); @@ -228,7 +232,7 @@ TEST_F(LIEstimatorPhaseTest, AllPathsToMain) { mcgm.addEdge(child1, grandchild); mcgm.addEdge(child2, grandchild); - for (CgNodePtr n : mcgm.getCallgraph()) { + for (CgNodePtr n : *mcgm.getCallgraph()) { n->get()->setNumberOfStatements(100); } @@ -238,7 +242,7 @@ TEST_F(LIEstimatorPhaseTest, AllPathsToMain) { grandchild->get()->setCallData(child2, 1, 10.0, 1.0, 0, 0); grandchild->get()->setCallData(child2, 1, 10.0, 100.0, 0, 1); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); // apply estimator phases auto liConfig = std::make_unique(LoadImbalance::LIConfig{ @@ -261,7 +265,8 @@ TEST_F(LIEstimatorPhaseTest, MajorPathsToMain) { Config cfg; auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); cm.removeAllEstimatorPhases(); cm.setConfig(&cfg); cm.setNoOutput(); @@ -284,7 +289,7 @@ TEST_F(LIEstimatorPhaseTest, MajorPathsToMain) { mcgm.addEdge(child1, grandchild); mcgm.addEdge(child2, grandchild); - for (CgNodePtr n : mcgm.getCallgraph()) { + for (CgNodePtr n : *mcgm.getCallgraph()) { n->get()->setNumberOfStatements(100); } @@ -294,7 +299,7 @@ TEST_F(LIEstimatorPhaseTest, MajorPathsToMain) { grandchild->get()->setCallData(child2, 1, 10.0, 1.0, 0, 0); grandchild->get()->setCallData(child2, 1, 10.0, 100.0, 0, 1); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); // apply estimator phases auto liConfig = std::make_unique( @@ -317,7 +322,8 @@ TEST_F(LIEstimatorPhaseTest, MajorParentSteps) { Config cfg; auto &cm = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - mcgm.reset(); + mcgm.resetManager(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); cm.removeAllEstimatorPhases(); cm.setConfig(&cfg); cm.setNoOutput(); @@ -346,7 +352,7 @@ TEST_F(LIEstimatorPhaseTest, MajorParentSteps) { child3->get()->setCallData(mainNode, 1, 10.0, 1.0, 0, 0); child3->get()->setCallData(mainNode, 1, 10.0, 100.0, 0, 1); - cm.setCG(mcgm.getCallgraph()); + cm.setCG(*mcgm.getCallgraph()); // apply estimator phases auto liConfig = std::make_unique(LoadImbalance::LIConfig{ LoadImbalance::MetricType::Efficiency, 1.2, 0.1, LoadImbalance::ContextStrategy::MajorParentSteps, 1, diff --git a/pgis/tool/PGISMain.cpp b/pgis/tool/PGISMain.cpp index 4aa9a20..f180b96 100644 --- a/pgis/tool/PGISMain.cpp +++ b/pgis/tool/PGISMain.cpp @@ -249,7 +249,7 @@ int main(int argc, char **argv) { float runTimeThreshold = .0f; auto &cg = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); - + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); cg.setConfig(&c); cg.setExtrapConfig(parseExtrapArgs(result)); @@ -309,8 +309,8 @@ int main(int argc, char **argv) { mcgReader.read(mcgm); } - spdlog::get("console")->info("Read MetaCG with {} nodes.", mcgm.getCallgraph().size()); - cg.setCG(mcgm.getCallgraph()); + spdlog::get("console")->info("Read MetaCG with {} nodes.", mcgm.getCallgraph()->size()); + cg.setCG(*mcgm.getCallgraph()); if (applyStaticFilter) { // load imbalance detection diff --git a/pgis/tool/StmtPrinterMain.cpp b/pgis/tool/StmtPrinterMain.cpp index 3b15916..c2a3a76 100644 --- a/pgis/tool/StmtPrinterMain.cpp +++ b/pgis/tool/StmtPrinterMain.cpp @@ -114,6 +114,7 @@ int main(int argc, char **argv) { // PiraMCGProcessor cg(&c); auto &cg = metacg::pgis::PiraMCGProcessor::get(); auto &mcgm = metacg::graph::MCGManager::get(); + mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); cg.setConfig(&c); if (stringEndsWith(filePath_ipcg, ".ipcg")) { @@ -123,7 +124,7 @@ int main(int argc, char **argv) { mcgReader.read(mcgm); // XXX Removable after refactoring - cg.setCG(mcgm.getCallgraph()); + cg.setCG(*mcgm.getCallgraph()); // XXX if (argc == 2) { @@ -159,7 +160,7 @@ int main(int argc, char **argv) { std::cout << "Registered estimator phases.\n"; // XXX Removable after refactoring - cg.setCG(mcgm.getCallgraph()); + cg.setCG(*mcgm.getCallgraph()); // XXX cg.applyRegisteredPhases(); } From fc40fe51161ad0b1b1b4723560ee3446ece5c098 Mon Sep 17 00:00:00 2001 From: "Lehr, Jan-Patrick" Date: Fri, 1 Apr 2022 13:24:58 +0200 Subject: [PATCH 10/12] Adds different classes to consolidate the properties of MetaCG file format versions and moves library tests to graph folder. --- .gitlab-ci.yml | 15 +++- cgcollector/tools/CGMerge.cpp | 2 +- formatSrc.sh | 4 + graph/CMakeLists.txt | 4 + graph/include/MCGBaseInfo.h | 95 ++++++++++++++++++++ graph/include/MCGReader.h | 3 - graph/include/MCGWriter.h | 46 +++++----- graph/include/Util.h | 43 ++++++++- graph/src/MCGReader.cpp | 52 ++++++----- graph/src/MCGWriter.cpp | 58 +++++------- graph/test/unit/CMakeLists.txt | 16 ++++ {pgis => graph}/test/unit/MCGManagerTest.cpp | 2 +- {pgis => graph}/test/unit/MCGReaderTest.cpp | 4 +- {pgis => graph}/test/unit/MCGWriterTest.cpp | 8 +- graph/test/unit/UtilTest.cpp | 45 ++++++++++ pgis/test/unit/CMakeLists.txt | 13 ++- 16 files changed, 307 insertions(+), 103 deletions(-) create mode 100644 graph/include/MCGBaseInfo.h create mode 100644 graph/test/unit/CMakeLists.txt rename {pgis => graph}/test/unit/MCGManagerTest.cpp (98%) rename {pgis => graph}/test/unit/MCGReaderTest.cpp (99%) rename {pgis => graph}/test/unit/MCGWriterTest.cpp (98%) create mode 100644 graph/test/unit/UtilTest.cpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1eb5362..9ee65a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,7 +62,8 @@ mcg-container: script: - podman login ${CI_REGISTRY} -u ${CI_CONTAINER_REG_USER} -p ${CONTAINER_REGISTRY_READ_TOKEN} - podman build -t registry.git.rwth-aachen.de/tuda-sc/projects/metacg/metacg:$CI_COMMIT_SHA -f container/metacg . - - podman run --rm -t metacg:$CI_COMMIT_SHA /opt/metacg/metacg/build/pgis/test/unit/basictests + - podman run --rm -t metacg:$CI_COMMIT_SHA /opt/metacg/metacg/build/pgis/test/unit/pgistests + - podman run --rm -t metacg:$CI_COMMIT_SHA /opt/metacg/metacg/build/graph/test/unit/libtests - podman image rm metacg:$CI_COMMIT_SHA build-mcg: @@ -119,6 +120,16 @@ install-mcg: - /tmp/metacg/bin/cgcollector --help - rm -r /tmp/metacg +test-graphlib: + stage: test + tags: + - general + before_script: *lb-setup + variables: + GIT_STRATEGY: none + GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_SHA + script: + - cd build/graph/test/unit && ./libtests test-pgis: stage: test @@ -129,7 +140,7 @@ test-pgis: GIT_STRATEGY: none GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_SHA script: - - cd build/pgis/test/unit && ./basictests + - cd build/pgis/test/unit && ./pgistests test-static-pgis: stage: integration-test diff --git a/cgcollector/tools/CGMerge.cpp b/cgcollector/tools/CGMerge.cpp index 0659af2..bb273da 100644 --- a/cgcollector/tools/CGMerge.cpp +++ b/cgcollector/tools/CGMerge.cpp @@ -47,7 +47,7 @@ nlohmann::json mergeFileFormatTwo(std::string wholeCGFilename, std::vector +#include + +namespace metacg { + +struct MCGFileFormatVersion { + MCGFileFormatVersion(int major, int minor) : major(major), minor(minor) {} + MCGFileFormatVersion(const MCGFileFormatVersion &other) = default; + MCGFileFormatVersion(MCGFileFormatVersion &&other) = default; + bool operator==(const MCGFileFormatVersion &rhs) const { return major == rhs.major && minor == rhs.minor; } + bool operator!=(const MCGFileFormatVersion &rhs) const { return !(rhs == *this); } + bool operator<(const MCGFileFormatVersion &rhs) const { + return major < rhs.major || (major == rhs.major && minor < rhs.minor); + } + bool operator>(const MCGFileFormatVersion &rhs) const { return !this->operator<(rhs); } + + [[nodiscard]] std::string getVersionStr() const { return std::to_string(major) + '.' + std::to_string(minor); } + + [[nodiscard]] std::string getJsonIdentifier() const { return {"version"}; } + + int major; + int minor; +}; + +struct MCGGeneratorVersionInfo { + MCGGeneratorVersionInfo(std::string &name, int major, int minor, std::string gitSHA = {}) + : name(name), major(major), minor(minor), sha(std::move(gitSHA)) {} + MCGGeneratorVersionInfo(const MCGGeneratorVersionInfo &other) = default; + MCGGeneratorVersionInfo(MCGGeneratorVersionInfo &&other) = default; + + [[nodiscard]] std::string getVersionStr() const { return std::to_string(major) + '.' + std::to_string(minor); } + + [[nodiscard]] std::string getJsonIdentifier() const { return {"generator"}; } + + [[nodiscard]] std::string getJsonNameIdentifier() const { return {"name"}; } + + [[nodiscard]] std::string getJsonVersionIdentifier() const { return {"version"}; } + + [[nodiscard]] std::string getJsonShaIdentifier() const { return {"sha"}; } + + std::string name; + int major; + int minor; + std::string sha; +}; + +struct MCGFileFormatInfo { + MCGFileFormatInfo(int major, int minor) : version(major, minor), cgFieldName("_CG"), metaInfoFieldName("_MetaCG") {} + MCGFileFormatInfo(const MCGFileFormatInfo &other) = default; + MCGFileFormatInfo(MCGFileFormatInfo &&other) = default; + + MCGFileFormatVersion version; + std::string cgFieldName; + std::string metaInfoFieldName; +}; + +struct MCGFileNodeInfo { + const std::string calleesStr{"callees"}; + const std::string isVirtualStr{"isVirtual"}; + const std::string doesOverrideStr{"doesOverride"}; + const std::string overridesStr{"overrides"}; + const std::string overriddenByStr{"overriddenBy"}; + const std::string callersStr{"callers"}; + const std::string hasBodyStr{"hasBody"}; + const std::string metaStr{"meta"}; +}; + +struct MCGFileInfo { + MCGFileInfo(MCGFileFormatInfo ffInfo, MCGGeneratorVersionInfo genInfo) + : formatInfo(std::move(ffInfo)), generatorInfo(std::move(genInfo)) {} + MCGFileInfo(const MCGFileInfo &other) = default; + MCGFileInfo(MCGFileInfo &&other) = default; + + MCGFileFormatInfo formatInfo; + MCGGeneratorVersionInfo generatorInfo; + MCGFileNodeInfo nodeInfo; +}; + +MCGFileInfo getVersionTwoFileInfo(MCGGeneratorVersionInfo mcgGenInfo); + +MCGGeneratorVersionInfo getCGCollectorGeneratorInfo(); + +} // namespace metacg + +#endif // METACG_MCGBASEINFO_H diff --git a/graph/include/MCGReader.h b/graph/include/MCGReader.h index f6bdcbd..4b355ac 100644 --- a/graph/include/MCGReader.h +++ b/graph/include/MCGReader.h @@ -168,9 +168,6 @@ class VersionTwoMetaCGReader : public MetaCGReader { void read(metacg::graph::MCGManager &cgManager) override; }; - - - } // namespace metacg::io #endif diff --git a/graph/include/MCGWriter.h b/graph/include/MCGWriter.h index 0880eca..484af4b 100644 --- a/graph/include/MCGWriter.h +++ b/graph/include/MCGWriter.h @@ -1,14 +1,15 @@ /** -* File: MCGWriter.h -* License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at -* https://github.com/tudasc/metacg/LICENSE.txt -*/ + * File: MCGWriter.h + * License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at + * https://github.com/tudasc/metacg/LICENSE.txt + */ #ifndef METACG_MCGWRITER_H #define METACG_MCGWRITER_H -#include "config.h" #include "Callgraph.h" +#include "MCGBaseInfo.h" +#include "config.h" #include "nlohmann/json.hpp" @@ -19,15 +20,9 @@ namespace metacg::io { */ class JsonSink { public: - explicit JsonSink() {} + void setJson(nlohmann::json jsonIn) { j = jsonIn; } - void setJson(nlohmann::json jsonIn) { - j = jsonIn; - } - - [[nodiscard]] const nlohmann::json & getJson() const { - return j; - } + [[nodiscard]] const nlohmann::json &getJson() const { return j; } /** * Outputs the Json stored in this sink into os and flushes. @@ -39,7 +34,7 @@ class JsonSink { } private: - nlohmann::json j; + nlohmann::json j{}; }; /** @@ -47,7 +42,9 @@ class JsonSink { */ class MCGWriter { public: - explicit MCGWriter(graph::MCGManager &mcgm) : mcgManager(mcgm) {} + explicit MCGWriter(graph::MCGManager &mcgm, + MCGFileInfo fileInfo = getVersionTwoFileInfo(getCGCollectorGeneratorInfo())) + : mcgManager(mcgm), fileInfo(std::move(fileInfo)) {} void write(JsonSink &js); @@ -56,13 +53,15 @@ class MCGWriter { * Adds the CG version data to the MetaCG in json Format. * @param j */ - inline void attachFormatTwoHeader(nlohmann::json &j) { - std::string cgcMajorVersion = std::to_string(CGCollector_VERSION_MAJOR); - std::string cgcMinorVersion = std::to_string(CGCollector_VERSION_MINOR); - std::string cgcVersion{cgcMajorVersion + '.' + cgcMinorVersion}; - j = {{"_MetaCG", {}}, {"_CG", {}}}; - j["_MetaCG"] = {{"version", "2.0"}, - {"generator", {{"name", "CGCollector"}, {"version", cgcVersion}, {"sha", MetaCG_GIT_SHA}}}}; + inline void attachMCGFormatHeader(nlohmann::json &j) { + const auto formatInfo = fileInfo.formatInfo; + const auto generatorInfo = fileInfo.generatorInfo; + j = {{formatInfo.metaInfoFieldName, {}}, {formatInfo.cgFieldName, {}}}; + j[formatInfo.metaInfoFieldName] = {{formatInfo.version.getJsonIdentifier(), formatInfo.version.getVersionStr()}, + {generatorInfo.getJsonIdentifier(), + {{generatorInfo.getJsonNameIdentifier(), generatorInfo.name}, + {generatorInfo.getJsonVersionIdentifier(), generatorInfo.getVersionStr()}, + {generatorInfo.getJsonShaIdentifier(), generatorInfo.sha}}}}; } /** @@ -79,6 +78,7 @@ class MCGWriter { void createAndAddMetaData(CgNodePtr node, const graph::MCGManager &mcgm, nlohmann::json &j); graph::MCGManager &mcgManager; + MCGFileInfo fileInfo; }; /* @@ -130,5 +130,5 @@ void annotateJSON(metacg::Callgraph &cg, const std::string &filename, PropRetrie } } -} // namespace metacg::io +} // namespace metacg::io #endif // METACG_MCGWRITER_H diff --git a/graph/include/Util.h b/graph/include/Util.h index 7847b96..9962ed1 100644 --- a/graph/include/Util.h +++ b/graph/include/Util.h @@ -10,10 +10,12 @@ #include "CgNode.h" #include #include +#include +#include namespace metacg::util { -std::set to_string(const std::set &nodes) { +inline std::set to_string(const std::set &nodes) { std::set names; for (const auto n : nodes) { names.emplace(n->getFunctionName()); @@ -21,6 +23,45 @@ std::set to_string(const std::set &nodes) { return names; } +inline std::vector string_split(const std::string &in, const char c = '.') { + std::vector positions; + positions.emplace_back(0); + while (auto pos = in.find(c, positions.back() + 1)) { + if (pos == std::string::npos) { + break; + } + positions.push_back(pos + 1); + if (positions.size() > 5000) { + exit(-1); + } + } + std::vector vec; + for (auto bg = positions.begin(); bg != positions.end(); ++bg) { + auto end = bg + 1; + vec.emplace_back(in.substr(*bg, (end - bg))); + } + return vec; +} + +inline int getVersionNoAtPosition(const std::string &versionStr, int index) { + auto numOccurrences = std::count(versionStr.begin(), versionStr.end(), '.'); + if (numOccurrences < 1) { + spdlog::get("errconsole")->error("Could not interpret version string"); + exit(-1); + } + auto versionParts = string_split(versionStr); + int version = std::atoi(versionParts.at(index).c_str()); + return version; +} + +inline int getMajorVersionFromString(const std::string &versionStr) { + return getVersionNoAtPosition(versionStr, 0); +} + +inline int getMinorVersionFromString(const std::string &versionStr) { + return getVersionNoAtPosition(versionStr, 1); +} + } #endif // METACG_UTIL_H diff --git a/graph/src/MCGReader.cpp b/graph/src/MCGReader.cpp index 733f1a4..2b494fd 100644 --- a/graph/src/MCGReader.cpp +++ b/graph/src/MCGReader.cpp @@ -1,8 +1,11 @@ /** * File: MCGReader.cpp - * License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at https://github.com/tudasc/metacg/LICENSE.txt + * License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at + * https://github.com/tudasc/metacg/LICENSE.txt */ #include "MCGReader.h" +#include "MCGBaseInfo.h" +#include "Util.h" #include "Timing.h" @@ -47,7 +50,7 @@ void MetaCGReader::buildGraph(metacg::graph::MCGManager &cgManager, MetaCGReader // Register nodes in the actual graph for (const auto &[k, fi] : functions) { console->trace("Inserting MetaCG node for function {}", k); - auto node = cgManager.findOrCreateNode(k); // node pointer currently unused + auto node = cgManager.findOrCreateNode(k); // node pointer currently unused node->setIsVirtual(fi.isVirtual); node->setHasBody(fi.hasBody); for (const auto &c : fi.callees) { @@ -190,7 +193,7 @@ void VersionOneMetaCGReader::addNumStmts(metacg::graph::MCGManager &cgm) { auto g = cgm.getCallgraph(); auto node = g->getNode(fi.functionName); assert(node != nullptr && "Nodes with #statements attached should be available"); - if (node->has()){ + if (node->has()) { auto pod = node->get(); pod->setNumberOfStatements(fi.numStatements); pod->setHasBody(fi.hasBody); @@ -205,30 +208,35 @@ void VersionOneMetaCGReader::addNumStmts(metacg::graph::MCGManager &cgm) { */ void VersionTwoMetaCGReader::read(metacg::graph::MCGManager &cgManager) { metacg::RuntimeTimer rtt("VersionTwoMetaCGReader::read"); + MCGFileFormatInfo ffInfo{2, 0}; auto j = source.get(); - auto mcgInfo = j["_MetaCG"]; + auto mcgInfo = j[ffInfo.metaInfoFieldName]; if (mcgInfo.is_null()) { spdlog::get("console")->error("Could not read version info from metacg file."); throw std::runtime_error("Could not read version info from metacg file"); } + /// XXX How to make that we can use the MCGGeneratorVersionInfo to access the identifiers auto mcgVersion = mcgInfo["version"].get(); auto generatorName = mcgInfo["generator"]["name"].get(); auto generatorVersion = mcgInfo["generator"]["version"].get(); + MCGGeneratorVersionInfo genVersionInfo{generatorName, metacg::util::getMajorVersionFromString(generatorVersion), + metacg::util::getMinorVersionFromString(generatorVersion), ""}; spdlog::get("console")->info("The metacg (version {}) file was generated with {} (version: {})", mcgVersion, generatorName, generatorVersion); - { // raii - std::string metaReadersStr; - int i = 1; - for (const auto mh : cgManager.getMetaHandlers()) { - metaReadersStr += std::to_string(i) + ") " + mh->toolName() + " "; - ++i; - } - spdlog::get("console")->info("Executing the meta readers: {}", metaReadersStr); + { // raii + std::string metaReadersStr; + int i = 1; + for (const auto mh : cgManager.getMetaHandlers()) { + metaReadersStr += std::to_string(i) + ") " + mh->toolName() + " "; + ++i; + } + spdlog::get("console")->info("Executing the meta readers: {}", metaReadersStr); } - auto jsonCG = j["_CG"]; + MCGFileInfo fileInfo{ffInfo, genVersionInfo}; + auto jsonCG = j[ffInfo.cgFieldName]; if (jsonCG.is_null()) { spdlog::get("console")->error("The call graph in the metacg file was null."); throw std::runtime_error("CG in metacg file was null."); @@ -241,35 +249,35 @@ void VersionTwoMetaCGReader::read(metacg::graph::MCGManager &cgManager) { /** Bi-directional graph information */ std::unordered_set callees; - setIfNotNull(callees, it, "callees"); + setIfNotNull(callees, it, fileInfo.nodeInfo.calleesStr); fi.callees = callees; std::unordered_set parents; - setIfNotNull(parents, it, "callers"); // Different name compared to version 1.0 + setIfNotNull(parents, it, fileInfo.nodeInfo.callersStr); // Different name compared to version 1.0 fi.parents = parents; /** Overriding information */ - setIfNotNull(fi.isVirtual, it, "isVirtual"); - setIfNotNull(fi.doesOverride, it, "doesOverride"); + setIfNotNull(fi.isVirtual, it, fileInfo.nodeInfo.isVirtualStr); + setIfNotNull(fi.doesOverride, it, fileInfo.nodeInfo.doesOverrideStr); std::unordered_set overriddenFunctions; - setIfNotNull(overriddenFunctions, it, "overrides"); + setIfNotNull(overriddenFunctions, it, fileInfo.nodeInfo.overridesStr); fi.overriddenFunctions.insert(overriddenFunctions.begin(), overriddenFunctions.end()); std::unordered_set overriddenBy; - setIfNotNull(overriddenBy, it, "overriddenBy"); + setIfNotNull(overriddenBy, it, fileInfo.nodeInfo.overriddenByStr); fi.overriddenBy.insert(overriddenBy.begin(), overriddenBy.end()); /** Information relevant for analysis */ - setIfNotNull(fi.hasBody, it, "hasBody"); + setIfNotNull(fi.hasBody, it, fileInfo.nodeInfo.hasBodyStr); } auto potentialTargets = buildVirtualFunctionHierarchy(cgManager); buildGraph(cgManager, potentialTargets); for (json::iterator it = jsonCG.begin(); it != jsonCG.end(); ++it) { - /** + /** * Pass each attached meta reader the current json object, to see if it has meta data * particular to that reader attached. */ - auto &jsonElem = it.value()["meta"]; + auto &jsonElem = it.value()[fileInfo.nodeInfo.metaStr]; if (!jsonElem.is_null()) { for (const auto metaHandler : cgManager.getMetaHandlers()) { if (jsonElem.contains(metaHandler->toolName())) { diff --git a/graph/src/MCGWriter.cpp b/graph/src/MCGWriter.cpp index 53d39b9..1619c4b 100644 --- a/graph/src/MCGWriter.cpp +++ b/graph/src/MCGWriter.cpp @@ -10,32 +10,18 @@ using FunctionNames = std::set; -//inline void insertNode(nlohmann::json &callgraph, const std::string &nodeName, const FunctionNames &callees, -// const FunctionNames &callers, const FunctionNames &overriddenBy, -// const FunctionNames &overriddenFunctions, const bool isVirtual, const bool doesOverride, -// const bool hasBody, int version) { -// if (version == 1) { -// callgraph[nodeName] = {{"callees", callees}, -// {"isVirtual", isVirtual}, -// {"doesOverride", doesOverride}, -// {"overriddenFunctions", overriddenFunctions}, -// {"overriddenBy", overriddenBy}, -// {"parents", callers}, -// {"hasBody", hasBody}}; -// } else if (version == 2) { -// callgraph["_CG"][nodeName] = {{"callees", callees}, -// {"isVirtual", isVirtual}, -// {"doesOverride", doesOverride}, -// {"overrides", overriddenFunctions}, -// {"overriddenBy", overriddenBy}, -// {"callers", callers}, -// {"hasBody", hasBody}}; -// } -//} +metacg::MCGFileInfo metacg::getVersionTwoFileInfo(metacg::MCGGeneratorVersionInfo mcgGenInfo) { + return {MCGFileFormatInfo(2, 0), mcgGenInfo}; +} + +metacg::MCGGeneratorVersionInfo metacg::getCGCollectorGeneratorInfo() { + std::string cgcCollName("CGCollector"); + return {cgcCollName, CGCollector_VERSION_MAJOR, CGCollector_VERSION_MINOR, MetaCG_GIT_SHA}; +} void metacg::io::MCGWriter::write(JsonSink &js) { nlohmann::json j; - attachFormatTwoHeader(j); + attachMCGFormatHeader(j); for (const auto &n : *mcgManager.getCallgraph()) { createNodeData(n, j); // general node data? @@ -57,22 +43,24 @@ void metacg::io::MCGWriter::createNodeData(const CgNodePtr node, nlohmann::json // const auto overrides = node->getOverrides(); // const auto overriddenBy = node->getOverriddenBy(); // const auto doesOverride = node->getDoesOverride(); - const bool doesOverride {false}; + const bool doesOverride{false}; const std::set overrides; const std::set overriddenBy; -// insertNode(j, funcName, callees, callers, overriddenBy, overrides, isVirtual, doesOverride, hasBody, 2); + // insertNode(j, funcName, callees, callers, overriddenBy, overrides, isVirtual, doesOverride, hasBody, 2); - j["_CG"][funcName] = {{"callees", metacg::util::to_string(callees)}, - {"isVirtual", isVirtual}, - {"doesOverride", doesOverride}, - {"overrides", overrides}, - {"overriddenBy", overriddenBy}, - {"callers", metacg::util::to_string(callers)}, - {"hasBody", hasBody}, - {"meta", nullptr}}; + const auto nodeInfo = fileInfo.nodeInfo; + j[fileInfo.formatInfo.cgFieldName][funcName] = {{nodeInfo.calleesStr, metacg::util::to_string(callees)}, + {nodeInfo.isVirtualStr, isVirtual}, + {nodeInfo.doesOverrideStr, doesOverride}, + {nodeInfo.overridesStr, overrides}, + {nodeInfo.overriddenByStr, overriddenBy}, + {nodeInfo.callersStr, metacg::util::to_string(callers)}, + {nodeInfo.hasBodyStr, hasBody}, + {nodeInfo.metaStr, nullptr}}; } -void metacg::io::MCGWriter::createAndAddMetaData(CgNodePtr node, const metacg::graph::MCGManager &mcgm, nlohmann::json &j) { +void metacg::io::MCGWriter::createAndAddMetaData(CgNodePtr node, const metacg::graph::MCGManager &mcgm, + nlohmann::json &j) { const auto funcName = node->getFunctionName(); const auto mdHandlers = mcgm.getMetaHandlers(); for (const auto mdh : mdHandlers) { @@ -80,6 +68,6 @@ void metacg::io::MCGWriter::createAndAddMetaData(CgNodePtr node, const metacg::g continue; } const auto mdJson = mdh->value(node); - j["_CG"][funcName]["meta"][mdh->toolName()] = mdJson; + j[fileInfo.formatInfo.cgFieldName][funcName][fileInfo.nodeInfo.metaStr][mdh->toolName()] = mdJson; } } diff --git a/graph/test/unit/CMakeLists.txt b/graph/test/unit/CMakeLists.txt new file mode 100644 index 0000000..6dc273f --- /dev/null +++ b/graph/test/unit/CMakeLists.txt @@ -0,0 +1,16 @@ +# Now simply link against gtest or gtest_main as needed. Eg +add_executable( + libtests + MCGManagerTest.cpp + MCGReaderTest.cpp + MCGWriterTest.cpp + UtilTest.cpp +) + +target_link_libraries(libtests gtest_main) +add_pgis(libtests) + +# add_library(ipcg) +target_link_libraries(libtests mcg) + +add_test(NAME libraryTests COMMAND libtests) diff --git a/pgis/test/unit/MCGManagerTest.cpp b/graph/test/unit/MCGManagerTest.cpp similarity index 98% rename from pgis/test/unit/MCGManagerTest.cpp rename to graph/test/unit/MCGManagerTest.cpp index b694fb1..6e02334 100644 --- a/pgis/test/unit/MCGManagerTest.cpp +++ b/graph/test/unit/MCGManagerTest.cpp @@ -6,7 +6,7 @@ #include "gtest/gtest.h" -#include "LoggerUtil.h" +#include "../../../pgis/test/unit/LoggerUtil.h" #include "MCGManager.h" #include "MetaDataHandler.h" diff --git a/pgis/test/unit/MCGReaderTest.cpp b/graph/test/unit/MCGReaderTest.cpp similarity index 99% rename from pgis/test/unit/MCGReaderTest.cpp rename to graph/test/unit/MCGReaderTest.cpp index 47b055e..8bf093a 100644 --- a/pgis/test/unit/MCGReaderTest.cpp +++ b/graph/test/unit/MCGReaderTest.cpp @@ -6,11 +6,12 @@ #include "gtest/gtest.h" -#include "LoggerUtil.h" +#include "../../../pgis/test/unit/LoggerUtil.h" #include "MCGManager.h" #include "MCGReader.h" #include "nlohmann/json.hpp" + using namespace metacg; using json = nlohmann::json; @@ -176,6 +177,7 @@ TEST(VersionTwoMetaCGReaderTest, EmptyCG) { mcgm.addToManagedGraphs("emptyGraph",std::make_unique()); metacg::io::JsonSource js(j); metacg::io::VersionTwoMetaCGReader mcgReader(js); + // "The call graph in the metacg file was null" ASSERT_THROW(mcgReader.read(mcgm), std::runtime_error); } diff --git a/pgis/test/unit/MCGWriterTest.cpp b/graph/test/unit/MCGWriterTest.cpp similarity index 98% rename from pgis/test/unit/MCGWriterTest.cpp rename to graph/test/unit/MCGWriterTest.cpp index 7f9b702..8676644 100644 --- a/pgis/test/unit/MCGWriterTest.cpp +++ b/graph/test/unit/MCGWriterTest.cpp @@ -5,7 +5,7 @@ */ #include "gtest/gtest.h" -#include "LoggerUtil.h" +#include "../../../pgis/test/unit/LoggerUtil.h" #include "MCGManager.h" #include "MCGWriter.h" #include "MetaDataHandler.h" @@ -39,7 +39,6 @@ static const std::string lid{"LIData"}; } // namespace JsonFieldNames TEST(MCGWriterTest, EmptyGraph) { - json j; loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); @@ -52,11 +51,11 @@ TEST(MCGWriterTest, EmptyGraph) { metacg::io::MCGWriter mcgw(mcgm); mcgw.write(jsSink); + auto j = jsSink.getJson(); ASSERT_TRUE(j[JsonFieldNames::cg].is_null()); } TEST(MCGWriterTest, OneNodeGraphNoHandlerAttached) { - json j; loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); @@ -79,7 +78,6 @@ TEST(MCGWriterTest, OneNodeGraphNoHandlerAttached) { } TEST(MCGWriterTest, OneNodeGraph) { - json j; loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); @@ -120,7 +118,6 @@ TEST(MCGWriterTest, OneNodeGraph) { } TEST(MCGWriterTest, OneNodeGraphWithData) { - json j; loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); @@ -170,7 +167,6 @@ TEST(MCGWriterTest, OneNodeGraphWithData) { } TEST(MCGWriterTest, OneNodeGraphNoDataForHandler) { - json j; loggerutil::getLogger(); auto &mcgm = metacg::graph::MCGManager::get(); diff --git a/graph/test/unit/UtilTest.cpp b/graph/test/unit/UtilTest.cpp new file mode 100644 index 0000000..38f431b --- /dev/null +++ b/graph/test/unit/UtilTest.cpp @@ -0,0 +1,45 @@ +/** + * File: UtilTest.cpp + * License: Part of the metacg project. Licensed under BSD 3 clause license. See LICENSE.txt file at + * https://github.com/tudasc/metacg/LICENSE.txt + */ + +#include "Util.h" +#include "gtest/gtest.h" + +TEST(UtilTest, string_split) { + std::string verStr{"0.2.0"}; + auto verParts = metacg::util::string_split(verStr); + ASSERT_EQ(verParts.size(), 3); + ASSERT_EQ(verParts[0], "0"); + ASSERT_EQ(verParts[1], "2"); + ASSERT_EQ(verParts[2], "0"); +} + +TEST(UtilTest, string_split2) { + std::string verStr{"1.0"}; + auto verParts = metacg::util::string_split(verStr); + ASSERT_EQ(verParts.size(), 2); + ASSERT_EQ(verParts[0], "1"); + ASSERT_EQ(verParts[1], "0"); +} + +TEST(UtilTest, getVersionNumberAtPosition) { + std::string verStr{"1.0"}; + auto verNumber = metacg::util::getVersionNoAtPosition(verStr, 0); + ASSERT_EQ(verNumber, 1); + verNumber = metacg::util::getVersionNoAtPosition(verStr, 1); + ASSERT_EQ(verNumber, 0); +} + +TEST(UtilTest, getMajorVersionNumber) { + std::string verStr{"1.0"}; + auto verNumber = metacg::util::getMajorVersionFromString(verStr); + ASSERT_EQ(verNumber, 1); +} + +TEST(UtilTest, getMinorVersionNumber) { + std::string verStr{"1.0"}; + auto verNumber = metacg::util::getMinorVersionFromString(verStr); + ASSERT_EQ(verNumber, 0); +} diff --git a/pgis/test/unit/CMakeLists.txt b/pgis/test/unit/CMakeLists.txt index 5dd8c09..b36883a 100644 --- a/pgis/test/unit/CMakeLists.txt +++ b/pgis/test/unit/CMakeLists.txt @@ -1,6 +1,6 @@ # Now simply link against gtest or gtest_main as needed. Eg add_executable( - basictests + pgistests CallgraphTest.cpp # CallgraphManagerTest.cpp CgNodeTest.cpp @@ -10,15 +10,12 @@ add_executable( loadImbalance/LIEstimatorPhaseTest.cpp loadImbalance/LIMetaDataTest.cpp loadImbalance/LIMetricTest.cpp - MCGManagerTest.cpp - MCGReaderTest.cpp - MCGWriterTest.cpp ) -target_link_libraries(basictests gtest_main) -add_pgis(basictests) +target_link_libraries(pgistests gtest_main) +add_pgis(pgistests) # add_library(ipcg) -target_link_libraries(basictests mcg) +target_link_libraries(pgistests mcg) -add_test(NAME cgnode_test COMMAND basictests) +add_test(NAME cgnode_test COMMAND pgistests) From f3272710228aeea86368bfe24bf02023c4b48f4d Mon Sep 17 00:00:00 2001 From: "Lehr, Jan-Patrick" Date: Fri, 1 Apr 2022 21:43:40 +0200 Subject: [PATCH 11/12] Improve CMake setup for third party libraries (and bumps json version) - Bumps nlohmann-json version to 3.10.5 - The CMake step accepts configure flags `METACG_USE_EXTERNAL_JSON` and `METACG_USE_EXTERNAL_SPDLOG` to enable detection. If those are not set to `ON`, the libraries are downloaded at configure time. --- CMakeLists.txt | 14 +++++++++ cmake/ToolchainOptions.cmake | 56 +++++++++++------------------------- cmake/json.cmake | 42 +++++++++++---------------- cmake/spdlog.cmake | 26 +++++++++++++++++ cmake/spdlog.cmake.in | 19 ------------ 5 files changed, 73 insertions(+), 84 deletions(-) create mode 100644 cmake/spdlog.cmake delete mode 100644 cmake/spdlog.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 46685cc..ee82fe9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,3 +85,17 @@ option( if(WITH_PGIS) add_subdirectory(pgis) endif() + +# If set to on, CMake looks for installed nlohmann::json +option( + METACG_USE_EXTERNAL_JSON + "On or off" + OFF +) + +# if set to on, CMake looks for installed spdlog +option( + METACG_USE_EXTERNAL_SPDLOG + "On or off" + OFF +) diff --git a/cmake/ToolchainOptions.cmake b/cmake/ToolchainOptions.cmake index e30e72c..d8895da 100644 --- a/cmake/ToolchainOptions.cmake +++ b/cmake/ToolchainOptions.cmake @@ -23,6 +23,7 @@ find_path(EXTRAP_INCLUDE extrap) include(json) include(cxxopts) +include(spdlog) # External dependencies function(add_clang target) @@ -51,10 +52,6 @@ function(add_extrap target) target_link_libraries(${target} extrap) endfunction() -function(add_spdlog_libraries target) - target_link_libraries(${target} spdlog::spdlog) -endfunction() - function(add_cube target) target_include_directories(${target} SYSTEM PUBLIC ${CUBE_INCLUDE}) target_link_directories( @@ -209,39 +206,18 @@ if(CMAKE_VERSION ) include_directories("${gtest_SOURCE_DIR}/include") endif() - -# Download and unpack spdlog at configure time -configure_file(cmake/spdlog.cmake.in spdlog-download/CMakeLists.txt) -execute_process( - COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . - RESULT_VARIABLE spdresult - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/spdlog-download -) -if(spdresult) - message(FATAL_ERROR "CMake step for spdlog failed: ${spdresult}") -endif() -execute_process( - COMMAND ${CMAKE_COMMAND} --build . - RESULT_VARIABLE spdresult - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/spdlog-download -) -if(spdresult) - message(FATAL_ERROR "Build step for spdlog failed: ${spdresult}") -endif() - -# Prevent overriding the parent project's compiler/linker settings on Windows -set(spdlog_force_shared_crt - ON - CACHE BOOL - "" - FORCE -) - -# Add spdlog directly to our build. -add_subdirectory( - ${CMAKE_CURRENT_BINARY_DIR}/spdlog-src - ${CMAKE_CURRENT_BINARY_DIR}/spdlog-build - EXCLUDE_FROM_ALL -) - -install(TARGETS spdlog DESTINATION lib) +# +# Download and unpack spdlog at configure time configure_file(cmake/spdlog.cmake spdlog-download/CMakeLists.txt) +# execute_process( COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . RESULT_VARIABLE spdresult WORKING_DIRECTORY +# ${CMAKE_CURRENT_BINARY_DIR}/spdlog-download ) if(spdresult) message(FATAL_ERROR "CMake step for spdlog failed: +# ${spdresult}") endif() execute_process( COMMAND ${CMAKE_COMMAND} --build . RESULT_VARIABLE spdresult WORKING_DIRECTORY +# ${CMAKE_CURRENT_BINARY_DIR}/spdlog-download ) if(spdresult) message(FATAL_ERROR "Build step for spdlog failed: +# ${spdresult}") endif() +# +# Prevent overriding the parent project's compiler/linker settings on Windows set(spdlog_force_shared_crt ON CACHE BOOL +# "" FORCE ) +# +# Add spdlog directly to our build. add_subdirectory( ${CMAKE_CURRENT_BINARY_DIR}/spdlog-src +# ${CMAKE_CURRENT_BINARY_DIR}/spdlog-build EXCLUDE_FROM_ALL ) +# +# install(TARGETS spdlog DESTINATION lib) diff --git a/cmake/json.cmake b/cmake/json.cmake index 7078f5c..e7c84da 100644 --- a/cmake/json.cmake +++ b/cmake/json.cmake @@ -1,31 +1,23 @@ -include(ExternalProject) +include(FetchContent) -if(DEFINED JSON_INCLUDE) - message("JSON_INCLUDE predefined: ${JSON_INCLUDE}") +if(METACG_USE_EXTERNAL_JSON) + message("Using externally found json library") + # Taken from https://cmake.org/cmake/help/v3.16/command/find_package.html#version-selection Should enable to use the + # highest available version number, should the package provide sorting + set(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL) + set(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC) + find_package( + nlohmann_json + 3.10 + REQUIRED + ) else() - find_path(JSON_LIBRARY NAMES json) - if(JSON_LIBRARY) - set(JSON_INCLUDE ${JSON_LIBRARY}/json/single_include) - message("JSON found in ${JSON_INCLUDE}") - else() - message("JSON library not found, download into extern during make") - ExternalProject_Add( - json - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/extern/json - GIT_REPOSITORY "https://github.com/nlohmann/json.git" - GIT_TAG master - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - GIT_SHALLOW true - ) - set(JSON_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/extern/json/single_include) - endif() + message("Using fetched release version of json library") + + FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.10.5/json.tar.xz) + FetchContent_MakeAvailable(json) endif() function(add_json target) - add_dependencies(${target} json) - - target_include_directories(${target} SYSTEM PUBLIC ${JSON_INCLUDE}) + target_link_libraries(${target} nlohmann_json::nlohmann_json) endfunction() diff --git a/cmake/spdlog.cmake b/cmake/spdlog.cmake new file mode 100644 index 0000000..424785a --- /dev/null +++ b/cmake/spdlog.cmake @@ -0,0 +1,26 @@ +include(FetchContent) + +if(METACG_USE_EXTERNAL_SPDLOG) + message("Using externally found spdlog library") + find_package( + spdlog + 1.8.2 + REQUIRED + ) +else() + message("Using fetched release version of spdlog library") + + FetchContent_Declare(spdlog URL https://github.com/gabime/spdlog/archive/refs/tags/v1.8.2.tar.gz) + FetchContent_MakeAvailable(spdlog) + + # Only install when spdlog is desired as automatically downloaded library + install( + TARGETS spdlog + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + ) +endif() + +function(add_spdlog_libraries target) + target_link_libraries(${target} spdlog::spdlog) +endfunction() diff --git a/cmake/spdlog.cmake.in b/cmake/spdlog.cmake.in deleted file mode 100644 index 60cc3ea..0000000 --- a/cmake/spdlog.cmake.in +++ /dev/null @@ -1,19 +0,0 @@ -cmake_minimum_required(VERSION 3.2) - -project(spdlog-download NONE) - -include(ExternalProject) -# cmake-format: off -ExternalProject_Add( - spdlog - GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG v1.8.2 - SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/spdlog-src" - BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/spdlog-build" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - CMAKE_ARGS -DSPDLOG_BUILD_SHARED=ON -) -# cmake-format: on From 18e864415dc0d86505c8dda82934f28e54201591 Mon Sep 17 00:00:00 2001 From: "Lehr, Jan-Patrick" Date: Tue, 5 Apr 2022 13:59:52 +0200 Subject: [PATCH 12/12] Integrate final changes for release v0.4.0 * Updates the README to contain more info w.r.t. versions and dependencies. * Removes download of dependencies git repositories (now handled in CMake). * Adds copy of Extra-P include files to install tree (allows simplifying CMake?). * Updates docker file. * Adds Sebastian as author. --- .gitlab-ci.yml | 2 +- AUTHORS | 1 + README.md | 38 ++++++++++++++++++++++++++++---------- build_submodules.sh | 42 +++++++++++++++++------------------------- container/metacg | 2 +- 5 files changed, 48 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ee65a7..b515c52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,7 +75,7 @@ build-mcg: GIT_STRATEGY: none GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_SHA script: - - cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/metacg -DCUBE_LIB=$(dirname $(which cube_info))/../lib -DCUBE_INCLUDE=$(dirname $(which cube_info))/../include/cubelib -DEXTRAP_INCLUDE=./extern/src/extrap/extrap-3.0/include -DEXTRAP_LIB=./extern/install/extrap/lib -DSPDLOG_BUILD_SHARED=ON -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON + - cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/metacg -DCUBE_LIB=$(dirname $(which cube_info))/../lib -DCUBE_INCLUDE=$(dirname $(which cube_info))/../include/cubelib -DEXTRAP_INCLUDE=./extern/install/extrap/include -DEXTRAP_LIB=./extern/install/extrap/lib -DSPDLOG_BUILD_SHARED=ON -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON - cmake --build build --parallel test-cgc: diff --git a/AUTHORS b/AUTHORS index 92548bc..33a7574 100644 --- a/AUTHORS +++ b/AUTHORS @@ -15,5 +15,6 @@ Constantin Völter Peter Arzt Julian Hindelang Jonas Rickert +Sebastian Kreutzer Tim Heldmann diff --git a/README.md b/README.md index 69e0716..0f761bc 100644 --- a/README.md +++ b/README.md @@ -51,31 +51,38 @@ Examples are empirically determined performance models, runtime measurements, or ## Requirements / Building MetaCG consists of the graph library, a CG construction tool, and an example analysis tool. -Currently, all components are built. +The graph library is always built. +The CG construction tool `CGCollector` can be disabled via the CMake option `-DWITH_CGCOLLECTOR=OFF`. +The example analysis tool `PGIS` can be disabled via the CMake option `-DWITH_PGIS=OFF`. -**Build Requirements** +**Build Requirements (for full build)** - Clang/LLVM version 10. -- nlohmann/json library [github](https://github.com/nlohmann/json) - Cube 4.5 [scalasca.org](https://www.scalasca.org/software/cube-4.x/download.html) +- Extra-P 3.0 [.tar.gz](http://apps.fz-juelich.de/scalasca/releases/extra-p/extrap-3.0.tar.gz) +- nlohmann/json library [github](https://github.com/nlohmann/json) - spdlog [github](https://github.com/gabime/spdlog) - cxxopts [github](https://github.com/jarro2783/cxxopts) -- Extra-P 3.0 [.tar.gz](http://apps.fz-juelich.de/scalasca/releases/extra-p/extrap-3.0.tar.gz) - PyQt5 - matplotlib -### Building the package +### Building -Currently, one first needs to build the dependencies for PGIS. -This can be done by running the provided convenience script, which we use in a Ubuntu 20.04 container and on a RedHat 8.5 (with certain LMod modules). -This will build and install the required software into `./deps/src` and `./deps/install`, respectively. -No guarantees that the script works on your machine. +Clang/LLVM (v 10) and the cube library are assumed to be available on the system. +Extra-P can be built using the `build_submodules.sh` script provided in the repository, though it is not tested outside of our CI system. +It builds and installs Extra-P into `./deps/src` and `./deps/install`, respectively. + +The `nlohmann-json`, `spdlog` and `cxxopts` library are downloaded at configure time, unless `-DMETACG_USE_EXTERNAL_JSON=ON` or `-DMETACG_USE_EXTERNAL_SPDLOG=ON` is given. +This will search for the respective libraries in the common CMake locations. +While CMake looks for `nlohmann-json` version 3.10., MetaCG should work starting from version 3.9.2. +For spdlog, we rely on version 1.8.2 -- other versions *may* work. ```{.sh} -?> cd pgis ?> ./build_submodules.sh ``` Thereafter, the package can be configured and built from the top-level CMake. +Change the `CMAKE_INSTALL_PREFIX` to where you want your MetaCG installation to live. +Providing `SPDLOG_SHARED=ON` is necessary to build the shared object version of spdlog and prevent linker errors. ```{.sh} ?> cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/metacg -DCUBE_LIB=$(dirname $(which cube_info))/../lib -DCUBE_INCLUDE=$(dirname $(which cube_info))/../include/cubelib -DEXTRAP_INCLUDE=./extern/src/extrap/extrap-3.0/include -DEXTRAP_LIB=./extern/install/extrap/lib -DSPDLOG_BUILD_SHARED=ON -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON @@ -84,6 +91,17 @@ Thereafter, the package can be configured and built from the top-level CMake. ?> cmake --install build ``` +### CMake Options + +- Path `CUBE_LIB`: Path to the libcube library directory +- Path `CUBE_INCLUDE`: Path to the libcube include directory +- Path `EXTRAP_LIB`: Path to the Extra-P library directory +- Path `EXTRAP_INCLUDE`: Path to the Extra-P include directory +- Bool `WITH_CGCOLLECTOR`: Whether to build call-graph construction tool +- Bool `WITH_PGIS`: Whether to build demo-analysis tool +- Bool `METACG_USE_EXTERNAL_JSON`: Search for installed version of nlohmann-json +- Bool `METACG_USE_EXTERNAL_SPDLOG`: Search for installed version of spdlog + ## Usage ### Graph Library diff --git a/build_submodules.sh b/build_submodules.sh index b68140a..3e431e0 100755 --- a/build_submodules.sh +++ b/build_submodules.sh @@ -1,8 +1,8 @@ #! /usr/bin/env bash #""" # File: build_submodules.sh -# License: Part of the PIRA project. Licensed under BSD 3 clause license. See LICENSE.txt file at https://github.com/jplehr/pira/LICENSE.txt -# Description: Helper script to build the git submodules useed in PIRA. +# License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at https://github.com/tudasc/MetaCG/LICENSE.txt +# Description: Helper script to build the git submodules useed in MetaCG. #""" scriptdir="$( @@ -20,12 +20,12 @@ parallel_jobs="$1" add_flags="$2" # Extra-P (https://www.scalasca.org/software/extra-p/download.html) -echo "[PIRA] Building Extra-P (for PIRA II modeling)" -echo "[PIRA] Getting prerequisites ..." +echo "[MetaCG] Building Extra-P (for PIRA II modeling)" +echo "[MetaCG] Getting prerequisites ..." pip3 install --user PyQt5 2>&1 >/dev/null pip3 install --user matplotlib 2>&1 >/dev/null if [ $? -ne 0 ]; then - echo "[PIRA] Installting Extra-P dependencies failed." + echo "[MetaCG] Installting Extra-P dependencies failed." exit 1 fi @@ -34,7 +34,7 @@ cd $extsourcedir/extrap # TODO check if extra-p is already there, if so, no download / no build? if [ ! -f "extrap-3.0.tar.gz" ]; then - echo "[PIRA] Downloading and building Extra-P" + echo "[MetaCG] Downloading and building Extra-P" wget http://apps.fz-juelich.de/scalasca/releases/extra-p/extrap-3.0.tar.gz fi tar xzf extrap-3.0.tar.gz @@ -46,35 +46,27 @@ mkdir build && cd build # but at least with python 3.9 on ubuntu it is a bit buggy and some distributions don't support it at all pythonheader=$(python3 -c "from sysconfig import get_paths; print(get_paths()[\"include\"])") if [ -z $pythonheader ]; then - echo "[PIRA] Python header not found." + echo "[MetaCG] Python header not found." exit 1 fi -echo "[PIRA] Found Python.h at " $pythonheader +echo "[MetaCG] Found Python.h at " $pythonheader ../configure --prefix=$extinstalldir/extrap CPPFLAGS=-I$pythonheader 2>&1 >/dev/null if [ $? -ne 0 ]; then - echo "[PIRA] Configuring Extra-P failed." + echo "[MetaCG] Configuring Extra-P failed." exit 1 fi + make -j $parallel_jobs 2>&1 >/dev/null if [ $? -ne 0 ]; then - echo "[PIRA] Building Extra-P failed." + echo "[MetaCG] Building Extra-P failed." exit 1 fi -make install 2>&1 >/dev/null -# CXX Opts -echo "[PIRA] Getting cxxopts library" -cd $extsourcedir -if [ ! -d "$extsourcedir/cxxopts" ]; then - git clone --branch 2_1 --depth 1 https://github.com/jarro2783/cxxopts cxxopts 2>&1 >/dev/null +make install 2>&1 >/dev/null +if [ $? -ne 0 ]; then + echo "[MetaCG] Installing Extra-P failed." + exit 1 fi -#cd cxxopts -#echo "[PIRA] Select release branch 2_1 for cxxopts." -#git checkout 2_1 2>&1 >/dev/null -# JSON library -echo "[PIRA] Getting json library" -cd $extsourcedir -if [ ! -d "$extsourcedir/json" ]; then - git clone --depth 1 --branch v3.9.1 https://github.com/nlohmann/json json 2>&1 >/dev/null -fi +mkdir $extinstalldir/extrap/include +cp $extsourcedir/extrap/extrap-3.0/include/* $extinstalldir/extrap/include diff --git a/container/metacg b/container/metacg index 749feca..1a5b853 100644 --- a/container/metacg +++ b/container/metacg @@ -8,5 +8,5 @@ COPY . /opt/metacg/metacg RUN cd metacg && \ PATH=/opt/metacg/extern/install/cubelib/bin:$PATH ./build_submodules.sh $(nproc) && \ - cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/metacg -DCUBE_LIB=/opt/metacg/extern/install/cubelib/lib -DCUBE_INCLUDE=/opt/metacg/extern/install/cubelib/include/cubelib -DEXTRAP_INCLUDE=./extern/src/extrap/extrap-3.0/include -DEXTRAP_LIB=./extern/install/extrap/lib -DSPDLOG_BUILD_SHARED=ON -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON && \ + cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/metacg -DCUBE_LIB=/opt/metacg/extern/install/cubelib/lib -DCUBE_INCLUDE=/opt/metacg/extern/install/cubelib/include/cubelib -DEXTRAP_INCLUDE=./extern/install/extrap/include -DEXTRAP_LIB=./extern/install/extrap/lib -DSPDLOG_BUILD_SHARED=ON -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON && \ cmake --build build --parallel