from concurrent.futures import ThreadPoolExecutor
from dag import DAG
import time
class Scheduler:
def __init__(self, dag):
self.dag = dag
def run_dag(self):
while not self.done():
runs = self.should_run()
for run in runs:
get_run(run)()
time.sleep(1)
def done(self, runs=None):
if not runs:
runs = self.dag.topological_sort()
return all(get_run(run).done()=='done' for run in runs)
def should_run(self):
ret = []
all_nodes = self.dag.topological_sort()
for node in all_nodes:
if get_run(node).done() == "pending" and (not self.dag.predecessors(node) or self.done(self.dag.predecessors(node))):
ret.append(node)
return ret
class MyDAG(DAG):
def __init__(self, executor):
super().__init__()
利用有向无环图DAG实现简单的任务依赖调度
最新推荐文章于 2024-08-23 08:13:20 发布
本文介绍了作者在学习Airflow后,尝试用有向无环图(DAG)自行实现简单任务调度的过程。文章提到,为提高效率,作者采用了py-dag库来辅助实现。
摘要由CSDN通过智能技术生成