Ticket #1623: duplicates.py

#!/usr/bin/env python

import os
import json
import difflib

with open(os.path.join('_locales', 'en_US', 'messages.json')) as file:
        msgs = [(msgid, data['message']) for msgid, data in json.load(file).items()]

while msgs:
        msgid, msgstr = msgs.pop(0)
        msgstr = msgstr.lower()

        group = [msgid]
        i = 0
        while i < len(msgs):
                msgid_, msgstr_ = msgs[i]
                msgstr_ = msgstr_.lower()

                if difflib.SequenceMatcher(None, msgstr, msgstr_).ratio() > 0.9:
                        group.append(msgid_)
                        del msgs[i]
                else:
                        i += 1
        
        if len(group) > 1:
                print group