update

Author: zhulifengsheng

README.md

@@ -20,6 +20,11 @@ Multi30k data from [here](https://github.com/multi30k/dataset) and [here](https:
 flickr30k entities data from [here](https://github.com/BryanPlummer/flickr30k_entities)  
 We get multi30k text data from [Revisit-MMT](https://github.com/LividWo/Revisit-MMT)
 ```bash
+cd fairseq_mmt
+git clone https://github.com/BryanPlummer/flickr30k_entities.git
+cd flickr30k_entities
+unzip annotations.zip
+
 # create a directory
 flickr30k
 ├─ flickr30k-images
@@ -52,6 +57,7 @@ pip3 install stanfordcorenlp
 wget https://nlp.stanford.edu/software/stanford-corenlp-latest.zip
 unzip stanford-corenlp-latest.zip
 cd fairseq_mmt
+cat data/multi30k/train.en data/multi30k/valid.en data/multi30k/test.2016.en > train_val_test2016.en
 python3 record_masking_position.py 
 
 cd data/masking

data/get_origin_en.py

@@ -2,6 +2,8 @@
 import os
 import shutil
 
+src-tgt = 'en-de'
+
 data_path = os.path.abspath(os.path.join(os.getcwd(), "../.."))
 
 train_lines = 29000

data/masking/create_masking1234_multi30k.py

@@ -1,7 +1,12 @@
 import os
+
+src-tgt = 'en-de'
+
 now_path = os.getcwd()
 if os.path.exists(os.path.join(now_path, 'data')):
     os.mkdir(os.path.join(now_path, 'data'))
+if os.path.exists(os.path.join(now_path, 'data', src-tgt)):
+    os.mkdir(os.path.join(now_path, 'data', src-tgt))
 
 data_path = os.path.abspath(os.path.join(os.getcwd(), "../.."))
 multi30k_dir = os.path.join(data_path, 'multi30k')
@@ -10,7 +15,7 @@
 _list = []
 
 _f = open(os.path.join(multi30k_dir, 'multi30k.en'), 'r', encoding='utf-8')
-with open(os.path.join(multi30k_dir, 'multi30k-en-de.bpe.en'), 'r', encoding='utf-8') as f:
+with open(os.path.join(multi30k_dir, 'multi30k-'+src-tgt+'.bpe.en'), 'r', encoding='utf-8') as f:
     for sentence_bpe, sentence in zip(f, _f):
         count += 1
         bpe =  sentence_bpe.strip().split()
@@ -37,7 +42,7 @@
 
             _list.append(dic)
 
-with open(os.path.join(now_path, 'data', 'origin2bpe.en-de.match'), 'w', encoding='utf-8') as f:
+with open(os.path.join(now_path, 'data', src-tgt, 'origin2bpe.'+src-tgt+'.match'), 'w', encoding='utf-8') as f:
     for i in _list:
         if isinstance(i, int):
            f.write(str(-1)+'\n')

data/masking/data/en2de/origin2bpe.en-de.match renamed to data/masking/data/en-de/origin2bpe.en-de.match

@@ -3,47 +3,86 @@
 from stanfordcorenlp import StanfordCoreNLP
 nlp = StanfordCoreNLP(r'../stanford-corenlp-4.3.2')#, lang='de')
 
-with open('train_val_test2016.en','r') as f:
-    sentences = f.readlines()
-sentence_list = []
-for i in sentences:
-    sentence_list.append(i.strip())
+def get_sentence_list():
+    sentence_list = []
+    with open('train_val_test2016.en','r') as f:
+        for l in f:
+            sentence_list.append(l.strip())
+    return sentence_list
 
-with open('train_val_test2016.txt','r') as f:
-    image_names = f.readlines()
-name_list = []
-for i in image_names:
-    name_list.append(i.strip())
+def filter_EscapeString(l):
+    l = l.replace(''', '\'')
+    l = l.replace("&", "&")
+    l = l.replace("& amp ;", '&')
+    l = l.replace(""", '"')
+    return l
 
-noun = defaultdict(int)
-nouns = defaultdict(int)
+def get_name_list():
+    name_list = []
+    with open('train_val_test2016.txt','r') as f:
+        for i in f:
+            name_list.append(i.split('.')[0])   
+    return name_list
+
+def fix_post_tag(phrase_pos_tag, phrase):
+    tmp = []
+    tmp_idx = 0
+    words = phrase.split()
+    for idx, i in enumerate(words):
+        if i == phrase_pos_tag[tmp_idx][0]:
+            tmp.append(phrase_pos_tag[tmp_idx])
+            tmp_idx += 1
+        else:
+            str1 = phrase_pos_tag[tmp_idx][0]
+            tmp_idx += 1
+            while str1 != i:
+                str1 += phrase_pos_tag[tmp_idx][0]
+                tmp_idx += 1
+            tmp.append((i, 'UNK'))
+    return tmp
+
+def write_dict(filename, dic):
+    out = open(filename, 'w', encoding='utf-8')
+    t = sorted(dic.items(), key=lambda item:item[1])
+    for i in t:
+        out.write(i[0] + ' ' + i[1])
+    out.close()
 
 if __name__ == "__main__":
+    noun = defaultdict(int)
+    nouns = defaultdict(int)
+    #people = defaultdict(int)
+    name_list = get_name_list()
+    sentence_list = get_sentence_list()
+ 
     for index in range(len(name_list)):
         image = name_list[index]
-        sentence = sentence_list[index]
-        x = get_sentence_data('../flickr30k_entities/Sentences/'+image.split('.')[0]+'.txt')
-        flag = True
-        for j in x:	# all matched
-            if j['sentence'].replace(' ','').replace('”','"').replace('`', '\'').replace('"', '') == sentence.replace('"', '').replace(' ', ''):
+        origin_sentence = sentence_list[index]
+        sentence = filter_EscapeString(origin_sentence)
+
+        # a list
+        x = get_sentence_data('flickr30k_entities/Sentences/'+image.split('.')[0]+'.txt')
+
+        for j in x:
+            entity_sentence = j['sentence'].replace(' ','').replace('”','"').replace('`','\'').replace('"','').lower()
+            if entity_sentence == sentence.replace('"','').replace(' ',''):
                 for t in j['phrases']:
-                    phrase = t['phrase']#.lower()
+                    phrase = t['phrase'].lower()
+                    # if 'people' in t['phrase_type']:
                     try:
-                        phrase_pos = nlp.pos_tag(phrase)
+                        phrase_pos_tag = nlp.pos_tag(phrase)
+                        if len(phrase_pos_tag) > len(phrase.split()):
+                            fix_post_tag(phrase_pos_tag, phrase)
+                        assert len(phrase_pos_tag) == len(phrase.split()):
                     except:
                         print(phrase)
-                    for pos in phrase_pos:
-                        if pos[1] == 'NN':
-                            noun[pos[0]] += 1
-                        elif pos[1] == 'NNS':
-                            nouns[pos[0]] += 1
-                flag = False
+
+                    #for pos_tag in phrase_pos_tag:
+                    #   if pos_tag[1] == 'NN':
+                    #       noun[pos_tag[0]] += 1
+                    #   elif pos_tag[1] == 'NNS':
+                    #       nouns[pos_tag[0]] += 1
                 break
-        if flag:
-            print(sentence)
-            for j in x:
-                print(j['sentence'].lower())
-            print()
-
-    print(len(noun))
-    print(len(nouns))
+
+    write_dict('data/masking/noun.en', noun)
+    write_dict('data/masking/nouns.en', nouns)