diff --git a/manager.py b/manager.py index 5f4a6a9d254fe5c3e146ac1062283af5250705e5..d2e840c8def28f1650bc97b0a4ccff837caf3f7d 100755 --- a/manager.py +++ b/manager.py @@ -8,30 +8,31 @@ import configparser import psutil import subprocess from workerapi import WorkerApi +import json config = configparser.ConfigParser() config.read('config.ini') scheduler = sched.scheduler() def run_scheduler(): - time.sleep(1) # weird things on startup - while True: - scheduler.run() - time.sleep(1) + time.sleep(1) # weird things on startup + while True: + scheduler.run() + time.sleep(1) def sched_func(delay, priority=0, firstdelay=None, args=[], kargs={}): - if firstdelay == None: - firstdelay = random.randint(1, 10) - def wrapper(func): - def sched_wrapper(): - try: - func(*args, **kargs) - except Exception: - traceback.print_exc() - scheduler.enter(delay, priority, sched_wrapper) - scheduler.enter(firstdelay, priority, sched_wrapper) - return func - return wrapper + if firstdelay == None: + firstdelay = random.randint(1, 10) + def wrapper(func): + def sched_wrapper(): + try: + func(*args, **kargs) + except Exception: + traceback.print_exc() + scheduler.enter(delay, priority, sched_wrapper) + scheduler.enter(firstdelay, priority, sched_wrapper) + return func + return wrapper api = WorkerApi(config['API']['BASE'],config['API']['KEY']) @@ -39,20 +40,28 @@ threading.Thread(target=run_scheduler, daemon=True).start() @sched_func(5) def ping_website_for_host(): - # ping so the website knows our host is still alive - if not api.worker_ping(): - print("Error sending host ping") + # ping so the website knows our host is still alive + if not api.worker_ping(): + print("Error sending host ping") -@sched_func(5) -def get_job(): - if psutil.cpu_percent(interval=3) > 85: - return - j = api.worker_schedule(config['JOBS']['TYPES']) - if not j: - return - print("id: %i, data: %s, all: %s"%(j['id'],j['data'],j)) - subprocess.Popen(['./worker.py', str(j['id']), str(j['type']), str(j['priority']) , str(j['data']) ] ) - print("forked") +procs = [] while True: - time.sleep(10) + for p in procs: + p.poll() + if p.returncode == None: + continue + print('done', p.returncode) + p.wait() + procs.remove(p) + if psutil.cpu_percent() > 85: + print("SLEEP CPU") + time.sleep(0.1) + continue + j = api.worker_schedule(config['JOBS']['TYPES']) + if not j: + print("SLEEP NOJOB") + time.sleep(1) + continue + print("id: %i, data: %s, all: %s"%(j['id'],j['data'],j)) + procs.append(subprocess.Popen(['./worker.py', str(j['id']), str(j['type']), str(j['priority']) , str(j['data']) ] ))