import sys
import os

rootdir="hdfs://haruna/home/byte_arnold_hl_mlnlc/user/wuxianze.0/Workspace/Multilingual/xgiga/en/checkpoints/mbartV2_iadapterfix*newTA*"

results = os.popen("hdfs dfs -ls -d {}".format(rootdir)).readlines()
for res in results:
    items = res.strip().split()
    directory = items[-1]
    inner_ls = os.popen("hdfs dfs -ls {}".format(directory)).readlines()
    # print(inner_ls)
    best_exists = any(["checkpoint_best.pt" in filename for filename in inner_ls])
    if best_exists:
        print("checkpoint_best.pt exists for Directory {}, delecting other files".format(directory))
        for ls_result in inner_ls:
            filepath = ls_result.strip().split()[-1]
            if not (filepath.endswith("checkpoint_best.pt") or filepath.endswith("checkpoint_last.pt")):
                # print(filepath)
                pipe = os.popen("hdfs dfs -rm {}".format(filepath))
                out = "".join(pipe.readlines())
                print(out)
