Spaces:
Sleeping
Sleeping
File size: 4,435 Bytes
c3ece9d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 |
from google_sheet import conf_repo, secret_repo
import json
from kaggle_service import KernelRerunService, NbJob
from logger import sheet_logger, get_now
from run_stt import run_stt_service
def get_secret_dict():
# load all account and secret into a dict
secret_dict = {}
try:
for i in range(2, 50): # note: read row 2 - 50
rs = secret_repo.read(i)
if not rs:
break
assert 'username' in rs
assert 'secret' in rs
username = rs['username'].rstrip()
secret = rs['secret'].rstrip()
secret_dict[username] = secret
except Exception as e:
sheet_logger.log(log="Get secret table failed!!" + str(e))
return secret_dict
def get_config_list(secret_dict):
configs = []
ids = []
try:
for i in range(2, 50): # note: read row 2 - 50
rs = conf_repo.read(i)
if not rs:
break
if not rs['config']:
reset_keys = []
for key in rs:
if rs[key]:
reset_keys.append(key)
if len(reset_keys) > 0:
conf_repo.update(row_index=i, data={k: '' for k in reset_keys})
break
print(i, rs)
try:
# validation
# assert 'config' in rs, 'require config column'
assert 'usernames' in rs, 'require usernames column'
assert 'enable' in rs, 'require enable column'
assert rs['enable'].rstrip() in ['enable', 'disable'], rs['enable']
assert 'slug' in rs['config'], 'require config.slug'
cfg = json.loads(rs['config'])
usernames = set(rs['usernames']
.rstrip().replace('\n', ' ').replace(',', ' ').replace(';', ' ').replace('|', ' ')
.split(' '))
usernames = [u for u in usernames if u]
is_enabled = rs['enable'].rstrip() == 'enable'
if not is_enabled:
sheet_logger.log(log="Disabled, skip!", nb=cfg['slug'])
print("skip ", cfg['slug'])
continue
assert len(usernames) > 0, 'len usernames == 0'
# process config
accounts = {u: secret_dict[u] for u in secret_dict if u in usernames}
assert not set(usernames).difference(set(accounts.keys())), set(usernames).difference(set(accounts.keys()))
cfg = {**cfg, "accounts": accounts}
# save
configs.append(cfg)
ids.append(i)
print(cfg)
except AssertionError:
import traceback
sheet_logger.update_job_status(i, validate_status=str(traceback.format_exc()))
except Exception:
import traceback
sheet_logger.log(log="Get config failed!!" + str(traceback.format_exc()))
return configs, ids
def create_service(configs, ids):
service = KernelRerunService()
for config, idx in zip(configs, ids):
try:
service.add_job(NbJob.from_dict(config, id=idx))
except Exception as e:
sheet_logger.update_job_status(idx, validate_status=str(e))
return service
def main():
sheet_logger.log("========= start ==========")
obj = run_stt_service.get_obj()
if run_stt_service.is_need_to_run(obj):
# start
run_stt_service.set_is_running({})
secret_dict = get_secret_dict()
# load config from gg sheet file
configs, ids = get_config_list(secret_dict)
# add config to service
service = create_service(configs, ids)
# run service
try:
service.validate_all()
# service.status_all()
service.run_all()
except Exception as e:
sheet_logger.log(log=str(e))
# done
run_stt_service.set_run_done()
else:
sheet_logger.log(f"Current time is {get_now()} is not after {obj.get('last_run')} {obj.get('auto_run_after_last_run')} or not require new run")
sheet_logger.log("========= end ==========")
if __name__ == "__main__":
main()
|