5_a_character_error_distance.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Distance between characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import mxnet as mx\n",
    "import difflib\n",
    "\n",
    "from ocr.handwriting_line_recognition import Network as BiLSTMNetwork, decode as topK_decode\n",
    "from ocr.utils.noisy_forms_dataset import Noisy_forms_dataset\n",
    "from ocr.utils.ngram_dataset import Ngram_dataset\n",
    "from ocr.utils.iam_dataset import resize_image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decode noisy forms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to find what characters are more likely to be confused with each others to build a distance model between them"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For that we do a diff of the predictions vs the form"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "line_image_size = (60, 800)\n",
    "def handwriting_recognition_transform(image):\n",
    "    image, _ = resize_image(image, line_image_size)\n",
    "    image = mx.nd.array(image)/255.\n",
    "    image = (image - 0.942532484060557) / 0.15926149044640417\n",
    "    image = image.as_in_context(ctx)\n",
    "    image = image.expand_dims(0).expand_dims(0)\n",
    "    return image\n",
    "\n",
    "def get_ns(is_train):\n",
    "    network = BiLSTMNetwork(rnn_hidden_states=512, rnn_layers=2, max_seq_len=160, ctx=ctx)\n",
    "    network.load_parameters(\"models/handwriting_line_sl_160_a_512_o_2.params\", ctx=ctx)\n",
    "\n",
    "    def noise_source_transform(image, text):\n",
    "        image = handwriting_recognition_transform(image)\n",
    "        output = network(image)\n",
    "        predict_probs = output.softmax().asnumpy()\n",
    "        return predict_probs\n",
    "    ns = Noisy_forms_dataset(noise_source_transform, train=is_train, name=\"OCR_noise2\", topK_decode=topK_decode)\n",
    "    return ns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ctx = mx.gpu(0) if mx.context.num_gpus() > 0 else mx.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ns = get_ns(is_train=True)\n",
    "ng_train_ds = Ngram_dataset(train_ns, \"word_5train\", output_type=\"word\", n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using ndiff to diff the expected result and the predicted results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "insertions = []\n",
    "deletions = []\n",
    "substitutions = []\n",
    "\n",
    "for i in range(len(ng_train_ds)):\n",
    "    _, _, noisy, actual = ng_train_ds[i]\n",
    "    diffs = []\n",
    "    for diff in difflib.ndiff(noisy, actual):\n",
    "        if diff[0] == \"+\" or diff[0] == \"-\":\n",
    "            diffs.append(diff)\n",
    "    if len(diffs) == 1:\n",
    "        if diffs[0][0] == \"+\":\n",
    "            insertions.append(diffs[0][-1])\n",
    "        if diffs[0][0] == \"-\":\n",
    "            deletions.append(diffs[0][-1])\n",
    "    if len(diffs) == 2:\n",
    "        if diffs[0][0] == \"+\" and diffs[1][0] == \"-\" or diffs[0][0] == \"-\" and diffs[1][0] == \"+\":\n",
    "            changes1 = (diffs[0][-1], diffs[1][-1])\n",
    "            changes2 = (diffs[1][-1], diffs[0][-1])\n",
    "            substitutions.append(changes1)\n",
    "            substitutions.append(changes2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using SequenceMatcher to diff the expected result and the predicted results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "insertions = []\n",
    "deletions = []\n",
    "substitutions = []\n",
    "output = []\n",
    "for i in range(len(ng_train_ds)):\n",
    "    _, _, noisy, actual = ng_train_ds[i]\n",
    "    seqm = difflib.SequenceMatcher(None, noisy, actual)\n",
    "    for opcode, a0, a1, b0, b1 in seqm.get_opcodes():\n",
    "        if opcode == 'equal':\n",
    "            output.append(seqm.a[a0:a1])\n",
    "        elif opcode == 'insert':\n",
    "            for char in seqm.b[b0:b1]:\n",
    "                insertions.append(char)\n",
    "        elif opcode == 'delete':\n",
    "            for char in seqm.a[a0:a1]:\n",
    "                deletions.append(char)\n",
    "        elif opcode == 'replace':\n",
    "            # seqm.a[a0:a1] -> seqm.b[b0:b1]\n",
    "            if len(seqm.a[a0:a1]) == len(seqm.b[b0:b1]):\n",
    "                for charA, charB in zip(seqm.a[a0:a1], seqm.b[b0:b1]):\n",
    "                    substitutions.append((charA, charB))\n",
    "        else:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      " 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  0.9 0.9 1.\n",
      " 1.  1.  0.8 0.8 0.8 0.8 1.  1.  0.8 1.  0.8 1.  0.9 0.8 1.  0.9 1.  0.9\n",
      " 0.9 0.9 1.  0.9 0.8 0.8 1.  1.  1.  0.9 1.  0.8 0.9 0.8 0.8 0.8 0.9 0.8\n",
      " 0.8 0.8 0.9 0.8 0.8 0.9 0.9 0.9 0.9 0.9 0.8 0.8 0.8 0.9 0.9 0.8 1.  0.9\n",
      " 1.  1.  1.  1.  1.  1.  1.  0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.9 0.8\n",
      " 0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.8 0.9 0.8 1.  1.  1.  1.\n",
      " 1.  1. ]\n"
     ]
    }
   ],
   "source": [
    "insertion_dict = {}\n",
    "for insertion in insertions:\n",
    "    if insertion not in insertion_dict:\n",
    "        insertion_dict[insertion] = 0\n",
    "    insertion_dict[insertion] += 1\n",
    "insertion_costs = np.ones(128, dtype=np.float64)\n",
    "for key in insertion_dict:\n",
    "    insertion_costs[ord(key)] = 0.9 if insertion_dict[key] <= 4 else 0.8\n",
    "print(insertion_costs)\n",
    "np.savetxt(\"models/insertion_costs.txt\", insertion_costs, fmt='%4.6f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'h': 40, 'r': 22, 'i': 17, 'W': 3, 'y': 8, 't': 51, 'n': 21, 'l': 14, 'e': 39, 'a': 23, 'A': 7, 's': 24, '.': 8, 'H': 2, 'u': 6, 'o': 14, 'm': 13, 'p': 4, 'S': 2, 'w': 20, 'x': 1, 'F': 3, 'T': 9, '1': 12, '5': 11, 'c': 12, 'M': 5, 'f': 2, 'G': 2, 'b': 4, 'g': 1, 'd': 8, ',': 3, '0': 1, 'B': 2, 'C': 3, '\"': 1, 'I': 1, 'v': 1}\n",
      "[1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      " 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  0.9 1.\n",
      " 1.  1.  1.  1.  1.  1.  1.  1.  0.9 1.  0.8 1.  0.9 0.8 1.  1.  1.  0.8\n",
      " 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  0.8 0.9 0.9 1.  1.  0.9 0.9\n",
      " 0.9 0.9 1.  1.  1.  0.8 1.  1.  1.  1.  1.  0.9 0.8 1.  1.  0.9 1.  1.\n",
      " 1.  1.  1.  1.  1.  1.  1.  0.8 0.9 0.8 0.8 0.8 0.9 0.9 0.8 0.8 1.  1.\n",
      " 0.8 0.8 0.8 0.8 0.9 1.  0.8 0.8 0.8 0.8 0.9 0.8 0.9 0.8 1.  1.  1.  1.\n",
      " 1.  1. ]\n"
     ]
    }
   ],
   "source": [
    "deletion_dict = {}\n",
    "for deletion in deletions:\n",
    "    if deletion not in deletion_dict:\n",
    "        deletion_dict[deletion] = 0\n",
    "    deletion_dict[deletion] += 1\n",
    "print(deletion_dict)\n",
    "deletion_costs = np.ones(128, dtype=np.float64)\n",
    "for key in deletion_dict:\n",
    "    deletion_costs[ord(key)] = 0.9 if deletion_dict[key] <= 4 else 0.8\n",
    "print(deletion_costs)\n",
    "np.savetxt(\"models/deletion_costs.txt\", deletion_costs, fmt='%4.6f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{('r', 's'): 5, ('l', 't'): 8, ('t', 'h'): 5, ('t', 'l'): 13, ('n', 'm'): 18, ('M', 'U'): 1, ('f', 't'): 1, ('A', 'N'): 1, ('e', 'o'): 13, ('e', 'u'): 2, ('n', 'r'): 9, ('h', 'k'): 4, ('e', 'a'): 18, ('c', 'e'): 3, ('.', ','): 21, ('H', 'M'): 1, ('c', 'C'): 3, ('t', 'r'): 4, ('L', 'h'): 1, ('W', 'b'): 1, ('r', 'e'): 3, ('r', 'R'): 1, ('r', 'n'): 10, ('r', 'v'): 5, ('P', 'R'): 1, ('o', 'e'): 6, ('v', 'r'): 4, ('t', 'd'): 4, ('n', 'a'): 1, ('h', 'L'): 1, ('W', 'S'): 1, ('W', 'w'): 3, ('r', 'x'): 2, ('c', 't'): 3, ('C', 'G'): 1, ('L', 't'): 1, ('a', 'b'): 1, ('e', 'M'): 3, ('y', 'g'): 6, ('e', 'm'): 1, ('a', 'o'): 24, ('S', 'I'): 1, ('r', 'i'): 3, ('w', 's'): 2, ('j', 'S'): 1, ('e', 'E'): 4, ('k', 'l'): 2, ('n', 't'): 2, ('t', 'k'): 2, ('e', 'w'): 1, ('h', '\"'): 1, ('t', 'M'): 1, ('\"', \"'\"): 6, (',', '.'): 13, ('w', 'a'): 1, ('l', 'L'): 2, ('l', 'h'): 3, ('e', 'n'): 3, ('u', 'n'): 3, ('f', 'F'): 1, ('f', 'P'): 1, ('t', 'n'): 1, ('l', 'n'): 1, ('n', 'u'): 5, ('o', 'a'): 6, ('t', 'f'): 4, ('W', 'I'): 1, ('t', 'b'): 2, ('w', 'I'): 1, ('l', 'k'): 1, ('c', 'o'): 2, ('t', 'H'): 2, ('s', 'o'): 2, ('c', 'r'): 1, ('a', 'e'): 6, ('i', 'a'): 1, ('a', 'A'): 9, ('o', 's'): 2, ('w', 'v'): 5, ('d', 'l'): 1, ('e', 'y'): 3, ('a', 'c'): 1, ('t', 'A'): 3, ('o', 'r'): 1, ('d', 'D'): 1, ('E', 'r'): 1, ('g', 'q'): 1, ('l', 's'): 2, ('S', 's'): 1, ('u', 'o'): 3, ('A', 'b'): 2, (',', ';'): 5, ('a', 'n'): 2, ('t', 's'): 2, ('F', 'f'): 1, ('o', 'O'): 3, ('y', 'e'): 1, ('n', 'c'): 1, ('t', '.'): 1, ('k', 'x'): 1, ('A', 'I'): 1, ('c', 's'): 2, ('e', 'c'): 4, ('l', 'b'): 2, ('e', 's'): 2, ('M', 'l'): 2, ('L', 'R'): 1, ('t', 'T'): 4, ('o', 'y'): 1, ('m', 'n'): 5, ('3', '8'): 2, ('s', 'g'): 1, ('e', 'i'): 2, ('.', 'I'): 1, ('s', 'k'): 1, ('B', 'b'): 2, ('a', 'u'): 3, ('I', 'i'): 1, ('w', 't'): 2, ('h', 'b'): 2, ('M', 'H'): 1, ('u', 'i'): 1, ('T', 't'): 1, ('w', 'r'): 1, ('T', 'i'): 1, ('n', 's'): 4, ('s', 'r'): 1, ('.', ':'): 1, ('g', 'G'): 1, ('m', 'v'): 1, ('h', 'n'): 2, ('i', 'o'): 1, ('w', 'i'): 1, ('N', 'M'): 1, ('H', 't'): 1, ('1', 'S'): 1, ('a', 'i'): 2, ('e', 'k'): 2, ('n', 'h'): 1, ('9', '3'): 1, ('f', 'G'): 1, ('w', 'W'): 1, ('h', 'H'): 3, ('n', 'N'): 2, ('l', 'U'): 1, ('i', \"'\"): 1, ('M', 'L'): 1, ('k', 'c'): 1, ('e', 'f'): 1, ('v', 'w'): 1, ('k', 'd'): 1, ('t', 'e'): 1, ('n', 'v'): 1, ('s', 't'): 1, ('e', 'p'): 2, ('k', 'w'): 1, ('s', 'd'): 1, ('r', 'b'): 1, ('t', ')'): 1, ('e', 'l'): 1, ('f', 'p'): 1, ('o', 'i'): 2, ('c', 'g'): 1, ('a', 't'): 3, ('t', 'z'): 1, ('i', 'e'): 1, ('p', 'b'): 4, ('s', 'S'): 1, ('r', 'w'): 1, ('t', 'c'): 1, ('C', 'c'): 1, ('E', '1'): 1, ('S', '5'): 1, ('s', 'c'): 1, ('z', 't'): 1, ('b', 'o'): 1, ('o', 'b'): 2, ('e', 't'): 1, ('a', 'l'): 2, ('r', 'z'): 1, ('i', 'u'): 1, ('o', 'u'): 1, ('c', 'd'): 1, ('c', 'p'): 1, ('G', 'c'): 1, ('M', 'n'): 1, ('p', 'r'): 1, ('B', 'h'): 1, ('s', 'n'): 2, ('x', 't'): 1, ('l', 'd'): 1, ('c', 'v'): 1, ('C', 'L'): 1, ('l', 'u'): 1, ('w', '\"'): 1, ('n', 'w'): 1, ('t', 'm'): 1}\n",
      "[[1. 1. 1. ... 1. 1. 1.]\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " ...\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " [1. 1. 1. ... 1. 1. 1.]]\n"
     ]
    }
   ],
   "source": [
    "substitution_dict = {}\n",
    "for subs in substitutions:\n",
    "    if subs not in substitution_dict:\n",
    "        substitution_dict[subs] = 0\n",
    "    substitution_dict[subs] += 1\n",
    "print(substitution_dict)\n",
    "substitute_costs = np.ones((128, 128), dtype=np.float64)\n",
    "for key in substitution_dict:\n",
    "    key1, key2 = key\n",
    "    substitute_costs[ord(key1), ord(key2)] = 0.9 if substitution_dict[key] <= 4 else 0.8\n",
    "print(substitute_costs)\n",
    "np.savetxt(\"models/substitute_costs.txt\", substitute_costs, fmt='%4.6f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}