with open('tokens.txt', 'r', encoding='utf-8') as f:
    words = set(line.strip() for line in f.readlines())

with open('roots.txt', 'w', encoding='utf-8') as f:
    for token in words:
        f.write(token+'\n')