40 ChrecMap_[&thetaNode].insert(std::move(chrec));
49 for (
auto & chrec : chrecs.Items())
52 if (
static_cast<int>(chrec->GetOperands().size()) == n + 1)
63 for (
const auto & [theta, chrecs] :
ChrecMap_)
65 total += chrecs.Size();
82 static std::unique_ptr<Context>
85 return std::make_unique<Context>();
89 std::unordered_map<const rvsdg::ThetaNode *, util::HashSet<std::unique_ptr<SCEVChainRecurrence>>>
122 static std::unique_ptr<Statistics>
125 return std::make_unique<Statistics>(sourceFile);
130 : Transformation(
"ScalarEvolution")
137 rvsdg::RvsdgModule & rvsdgModule,
138 util::StatisticsCollector & statisticsCollector)
140 auto statistics = Statistics::Create(rvsdgModule.SourceFilePath().value());
143 Context_ = Context::Create();
144 InductionVariableMap_.clear();
145 const rvsdg::Region & rootRegion = rvsdgModule.Rvsdg().GetRootRegion();
146 AnalyzeRegion(rootRegion);
148 statistics->Stop(*Context_);
149 statisticsCollector.CollectDemandedStatistics(std::move(statistics));
168 for (
const auto & node : region.
Nodes())
172 for (
auto & subregion : structuralNode->Subregions())
176 if (
const auto thetaNode =
dynamic_cast<const rvsdg::ThetaNode *
>(structuralNode))
179 for (
const auto loopVar : thetaNode->GetLoopVars())
181 Context_.get()->AddLoopVar(*loopVar.pre);
185 for (
auto & [output, chrec] : chrecMap)
188 Context_.get()->InsertChrec(*thetaNode, chrec);
195 std::optional<const SCEV *>
199 return it->second.get();
203 std::unique_ptr<SCEV>
208 return (*existing)->Clone();
211 std::unique_ptr<SCEV> result;
212 if (rvsdg::TryGetRegionParentNode<rvsdg::ThetaNode>(output))
216 result = std::make_unique<SCEVPlaceholder>(output);
218 if (
const auto simpleNode = rvsdg::TryGetOwnerNode<rvsdg::SimpleNode>(output))
220 if (rvsdg::is<IntegerConstantOperation>(simpleNode->GetOperation()))
225 result = std::make_unique<SCEVConstant>(value);
227 if (rvsdg::is<IntegerAddOperation>(simpleNode->GetOperation()))
230 const auto lhs = simpleNode->input(0)->origin();
231 const auto rhs = simpleNode->input(1)->origin();
232 result = std::make_unique<SCEVAddExpr>(
241 result = std::make_unique<SCEVUnknown>();
249 std::unordered_map<const rvsdg::Output *, int>
252 std::unordered_map<const rvsdg::Output *, int> dependencies{};
254 ||
dynamic_cast<const SCEVUnknown *
>(¤tSCEV))
258 if (
const auto placeholderSCEV =
dynamic_cast<const SCEVPlaceholder *
>(¤tSCEV))
260 if (
const auto dependency = placeholderSCEV->GetPrePointer())
261 dependencies[dependency]++;
263 if (
const auto addSCEV =
dynamic_cast<const SCEVAddExpr *
>(¤tSCEV))
266 std::unordered_map<const rvsdg::Output *, int> lhsDependencies =
268 std::unordered_map<const rvsdg::Output *, int> rhsDependencies =
272 for (
const auto & [ptr, count] : lhsDependencies)
273 dependencies[ptr] += count;
276 for (
const auto & [ptr, count] : rhsDependencies)
277 dependencies[ptr] += count;
289 const auto post = loopVar.post;
290 const auto scev =
UniqueSCEVs_.at(post->origin())->Clone();
292 const auto pre = loopVar.pre;
293 const std::unordered_map<const rvsdg::Output *, int> dependencies =
295 graph[pre] = dependencies;
302 std::vector<const rvsdg::Output *>
305 const size_t numVertices = dependencyGraph.size();
306 std::unordered_map<const rvsdg::Output *, int> indegree(numVertices);
307 std::queue<const rvsdg::Output *> q{};
308 for (
const auto & [node, deps] : dependencyGraph)
310 for (
const auto & dep : deps)
312 if (
const auto ptr = dep.first; ptr == node)
317 if (indegree[node] == 0)
324 std::vector<const rvsdg::Output *> result{};
329 result.push_back(currentNode);
331 for (
const auto & [node, deps] : dependencyGraph)
333 if (node == currentNode)
336 for (
const auto & dep : deps)
338 const auto ptr = dep.first;
341 if (ptr == currentNode)
345 if (indegree[node] == 0)
351 JLM_ASSERT(result.size() == numVertices &&
"Dependency graph can't contain cycles!");
360 if (dependencyGraph[&variable].size() >= 1)
365 std::unordered_map<const rvsdg::Output *, std::unique_ptr<SCEVChainRecurrence>>
370 const auto post = loopVar.post;
382 UniqueSCEVs_.insert_or_assign(loopVar.post->origin(), std::make_unique<SCEVUnknown>());
385 validIVs.insert(loopVar.pre);
390 auto filteredDependencyGraph = dependencyGraph;
391 for (
auto it = filteredDependencyGraph.begin(); it != filteredDependencyGraph.end();)
393 if (!validIVs.Contains(it->first))
395 for (
auto & [node, deps] : filteredDependencyGraph)
396 deps.erase(it->first);
397 it = filteredDependencyGraph.erase(it);
405 std::vector<const rvsdg::Output *> allVars{};
407 for (
const auto & indVarPre : order)
408 allVars.push_back(indVarPre);
411 if (std::find(order.begin(), order.end(), loopVar.pre) == order.end())
413 allVars.push_back(loopVar.pre);
416 for (
const auto loopVarPre : allVars)
418 if (std::find(order.begin(), order.end(), loopVarPre) != order.end())
421 const auto post = loopVar.
post;
422 const auto scev =
UniqueSCEVs_.at(post->origin()).get();
426 if (chainRecurrence->GetOperands().size() == 1
431 chainRecurrence.reset();
432 chainRecurrence = std::make_unique<SCEVChainRecurrence>(thetaNode);
440 chainRecurrence->AddOperandToFront(std::make_unique<SCEVConstant>(*constantInteger));
445 chainRecurrence->AddOperandToFront(std::make_unique<SCEVInit>(*loopVarPre));
451 auto unknownChainRecurrence = std::make_unique<SCEVChainRecurrence>(thetaNode);
452 unknownChainRecurrence->AddOperand(std::make_unique<SCEVUnknown>());
457 std::unordered_map<const rvsdg::Output *, std::unique_ptr<SCEVChainRecurrence>> chrecMap{};
463 chrecMap[loopVar.pre] = std::unique_ptr<SCEVChainRecurrence>(
470 std::unique_ptr<SCEVChainRecurrence>
473 const SCEV & scevTree,
476 auto chrec = std::make_unique<SCEVChainRecurrence>(thetaNode);
478 if (
const auto scevConstant =
dynamic_cast<const SCEVConstant *
>(&scevTree))
481 chrec->AddOperand(scevConstant->Clone());
484 if (
const auto scevPlaceholder =
dynamic_cast<const SCEVPlaceholder *
>(&scevTree))
486 if (scevPlaceholder->GetPrePointer() == &IV)
490 chrec->AddOperand(std::make_unique<SCEVConstant>(0));
498 return std::unique_ptr<SCEVChainRecurrence>(
501 chrec->AddOperand(std::make_unique<SCEVUnknown>());
504 if (
const auto scevAddExpr =
dynamic_cast<const SCEVAddExpr *
>(&scevTree))
509 const auto lhsSize = lhsChrec->GetOperands().size();
510 const auto rhsSize = rhsChrec->GetOperands().size();
511 for (
size_t i = 0; i < std::max(lhsSize, rhsSize); ++i)
515 if (i <= lhsSize - 1)
516 lhsOperand = lhsChrec->GetOperand(i);
518 if (i <= rhsSize - 1)
519 rhsOperand = rhsChrec->GetOperand(i);
520 chrec->AddOperand(
ApplyFolding(lhsOperand, rhsOperand));
524 JLM_ASSERT(
false &&
"Unknown SCEV type in CreateChainRecurrence!");
534 std::unique_ptr<SCEV>
559 const auto lhsUnknown =
dynamic_cast<SCEVUnknown *
>(lhsOperand);
560 const auto rhsUnknown =
dynamic_cast<SCEVUnknown *
>(rhsOperand);
561 const auto lhsInit =
dynamic_cast<SCEVInit *
>(lhsOperand);
562 const auto rhsInit =
dynamic_cast<SCEVInit *
>(rhsOperand);
563 const auto lhsNAryAddExpr =
dynamic_cast<SCEVNAryAddExpr *
>(lhsOperand);
564 const auto rhsNAryAddExpr =
dynamic_cast<SCEVNAryAddExpr *
>(rhsOperand);
565 const auto lhsConstant =
dynamic_cast<SCEVConstant *
>(lhsOperand);
566 const auto rhsConstant =
dynamic_cast<SCEVConstant *
>(rhsOperand);
569 if (lhsUnknown || rhsUnknown)
572 return std::make_unique<SCEVUnknown>();
574 if (lhsInit && rhsInit)
577 return std::make_unique<SCEVNAryAddExpr>(lhsInit->Clone(), rhsInit->Clone());
579 if ((lhsInit && rhsNAryAddExpr) || (rhsInit && lhsNAryAddExpr))
582 const auto * init = lhsInit ? lhsInit : rhsInit;
583 auto * nAryAddExpr = lhsNAryAddExpr ? lhsNAryAddExpr : rhsNAryAddExpr;
584 nAryAddExpr->AddOperand(init->Clone());
585 return nAryAddExpr->Clone();
590 const auto * init = lhsInit ? lhsInit : rhsInit;
591 const auto * constant = lhsConstant ? lhsConstant : rhsConstant;
592 return std::make_unique<SCEVNAryAddExpr>(init->Clone(), constant->Clone());
594 if (lhsInit || rhsInit)
597 const auto * init = lhsInit ? lhsInit : rhsInit;
598 return init->Clone();
600 if (lhsNAryAddExpr && rhsNAryAddExpr)
603 for (
auto op : rhsNAryAddExpr->GetOperands())
605 lhsNAryAddExpr->AddOperand(op->Clone());
607 return lhsNAryAddExpr->Clone();
613 auto * nAryAddExpr = lhsNAryAddExpr ? lhsNAryAddExpr : rhsNAryAddExpr;
614 auto * constant = lhsConstant ? lhsConstant : rhsConstant;
615 nAryAddExpr->AddOperand(constant->Clone());
616 return nAryAddExpr->Clone();
618 if (lhsNAryAddExpr || rhsNAryAddExpr)
620 const auto * nAryAddExpr = lhsNAryAddExpr ? lhsNAryAddExpr : rhsNAryAddExpr;
621 return nAryAddExpr->Clone();
623 if (lhsConstant && rhsConstant)
626 const auto lhsValue = lhsConstant->GetValue();
627 const auto rhsValue = rhsConstant->GetValue();
629 return std::make_unique<SCEVConstant>(lhsValue + rhsValue);
631 if (lhsConstant || rhsConstant)
633 const auto * constant = lhsConstant ? lhsConstant : rhsConstant;
634 return constant->Clone();
636 return std::make_unique<SCEVUnknown>();
645 if (dependencyGraph[&variable][&variable] != 1)
649 std::unordered_set<const rvsdg::Output *> visited{};
650 std::unordered_set<const rvsdg::Output *> recursionStack{};
659 std::unordered_set<const rvsdg::Output *> & visited,
660 std::unordered_set<const rvsdg::Output *> & recursionStack)
662 visited.insert(¤tIV);
663 recursionStack.insert(¤tIV);
665 for (
const auto & [depPtr, depCount] : dependencyGraph[¤tIV])
668 if (depPtr == ¤tIV)
673 if (depPtr == &originalIV)
677 if (visited.find(depPtr) != visited.end())
685 recursionStack.erase(¤tIV);
692 if (
typeid(a) !=
typeid(b))
698 return ca->
GetValue() == cb->GetValue();
701 if (
auto * ia =
dynamic_cast<const SCEVInit *
>(&a))
703 auto * ib =
dynamic_cast<const SCEVInit *
>(&b);
710 if (
auto * aa =
dynamic_cast<const SCEVAddExpr *
>(&a))
720 if (cha->GetLoop() != chb->GetLoop())
722 if (cha->GetOperands().size() != chb->GetOperands().size())
724 for (
size_t i = 0; i < cha->GetOperands().size(); ++i)
735 if (naa->GetOperands().size() != nab->GetOperands().size())
737 for (
size_t i = 0; i < naa->GetOperands().size(); ++i)
const IntegerValueRepresentation & Representation() const noexcept
std::vector< const SCEV * > GetOperands() const
const rvsdg::Output * GetPrePointer() const
int GetNumOfChrecsWithOrder(const int n) const
std::vector< const rvsdg::Output * > LoopVars_
size_t GetNumTotalLoopVars() const
Context & operator=(const Context &)=delete
Context(const Context &)=delete
void AddLoopVar(const rvsdg::Output &var)
size_t GetNumOfTotalChrecs() const
Context(Context &&)=delete
Context & operator=(Context &&)=delete
void InsertChrec(const rvsdg::ThetaNode &thetaNode, std::unique_ptr< SCEVChainRecurrence > &chrec)
std::unordered_map< const rvsdg::ThetaNode *, util::HashSet< std::unique_ptr< SCEVChainRecurrence > > > ChrecMap_
static std::unique_ptr< Context > Create()
static std::unique_ptr< Statistics > Create(const util::FilePath &sourceFile)
~Statistics() noexcept override=default
void Stop(const Context &context) noexcept
std::optional< const SCEV * > TryGetSCEVForOutput(const rvsdg::Output &output)
static std::unordered_map< const rvsdg::Output *, int > FindDependenciesForSCEV(const SCEV ¤tSCEV, const rvsdg::Output ¤tIV)
std::unordered_map< const rvsdg::Output *, std::unique_ptr< SCEV > > UniqueSCEVs_
void AnalyzeRegion(const rvsdg::Region ®ion)
std::unique_ptr< SCEV > GetOrCreateSCEVForOutput(const rvsdg::Output &output)
static std::vector< const rvsdg::Output * > TopologicalSort(const IVDependencyGraph &dependencyGraph)
std::unordered_map< const rvsdg::Output *, std::unique_ptr< SCEVChainRecurrence > > ChainRecurrenceMap_
~ScalarEvolution() noexcept override
IVDependencyGraph CreateDependencyGraph(const rvsdg::ThetaNode &thetaNode) const
std::unordered_map< const rvsdg::Output *, std::unique_ptr< SCEVChainRecurrence > > PerformSCEVAnalysis(const rvsdg::ThetaNode &thetaNode)
static bool HasCycleThroughOthers(const rvsdg::Output ¤tIV, const rvsdg::Output &originalIV, IVDependencyGraph &dependencyGraph, std::unordered_set< const rvsdg::Output * > &visited, std::unordered_set< const rvsdg::Output * > &recursionStack)
std::unique_ptr< Context > Context_
static bool IsUnknown(const SCEVChainRecurrence &chrec)
std::unordered_map< const rvsdg::Output *, std::unordered_map< const rvsdg::Output *, int > > IVDependencyGraph
static bool IsValidInductionVariable(const rvsdg::Output &variable, IVDependencyGraph &dependencyGraph)
static bool StructurallyEqual(const SCEV &a, const SCEV &b)
static std::unique_ptr< SCEV > ApplyFolding(SCEV *lhsOperand, SCEV *rhsOperand)
std::unique_ptr< SCEVChainRecurrence > CreateChainRecurrence(const rvsdg::Output &IV, const SCEV &scevTree, const rvsdg::ThetaNode &thetaNode)
Represent acyclic RVSDG subgraphs.
NodeRange Nodes() noexcept
LoopVar MapPreLoopVar(const rvsdg::Output &argument) const
Maps variable at start of loop iteration to full varibale description.
std::vector< LoopVar > GetLoopVars() const
Returns all loop variables.
util::Timer & GetTimer(const std::string &name)
util::Timer & AddTimer(std::string name)
void AddMeasurement(std::string name, T value)
Global memory state passed between functions.
bool isNonZeroConstant(const SCEVConstant *c)
bool DependsOnLoopVariable(const rvsdg::Output &variable, ScalarEvolution::IVDependencyGraph &dependencyGraph)
std::optional< int64_t > tryGetConstantSignedInteger(const rvsdg::Output &output)
rvsdg::Input * post
Variable after iteration (output result from subregion).
static const char * NumFirstOrderRecurrences
static const char * NumSecondOrderRecurrences
static const char * Timer
static const char * NumTotalRecurrences
static const char * NumLoopVariablesTotal
static const char * NumConstantRecurrences
static const char * NumThirdOrderRecurrences