Supervised_Statistical_MT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Supervised_Statistical_MT.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/priyanshu2103/Sanskrit-Hindi-Machine-Translation/blob/main/Supervised_Statistical_MT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VoDWTjffnPgT"
      },
      "source": [
        "# **```Supervised Statistical Machine Translation```**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0gIch8-tnmSz"
      },
      "source": [
        "This notebook runs the Moses SMT system, following the official documentation. Moses is used to translate Sanskrit to English, which is then translated to Hindi using Google Translate API.\n",
        "\n",
        "NOTE: Moses training and tuning takes around 2-2.5 hrs to run "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KJH6-cD4dxAU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "14df1fad-321f-4a5c-cc87-229aaf8d0cd3"
      },
      "source": [
        "# Downloading Moses Binaries directly, as it is complicated to install moses by source \n",
        "!wget http://www.statmt.org/moses/RELEASE-4.0/binaries/ubuntu-17.04.tgz\n",
        "!tar -xvzf ubuntu-17.04.tgz\n",
        "!rm -rf ubuntu-17.04.tgz"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-11-22 12:47:51--  http://www.statmt.org/moses/RELEASE-4.0/binaries/ubuntu-17.04.tgz\n",
            "Resolving www.statmt.org (www.statmt.org)... 129.215.197.184\n",
            "Connecting to www.statmt.org (www.statmt.org)|129.215.197.184|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 120382471 (115M) [application/x-gzip]\n",
            "Saving to: ‘ubuntu-17.04.tgz’\n",
            "\n",
            "ubuntu-17.04.tgz    100%[===================>] 114.81M  2.28MB/s    in 96s     \n",
            "\n",
            "2020-11-22 12:49:34 (1.19 MB/s) - ‘ubuntu-17.04.tgz’ saved [120382471/120382471]\n",
            "\n",
            "ubuntu-17.04/\n",
            "ubuntu-17.04/training-tools/\n",
            "ubuntu-17.04/training-tools/mkcls\n",
            "ubuntu-17.04/training-tools/snt2cooc\n",
            "ubuntu-17.04/training-tools/merge_alignment.py\n",
            "ubuntu-17.04/training-tools/mgiza\n",
            "ubuntu-17.04/moses/\n",
            "ubuntu-17.04/moses/scripts/\n",
            "ubuntu-17.04/moses/scripts/analysis/\n",
            "ubuntu-17.04/moses/scripts/analysis/sg2dot.perl\n",
            "ubuntu-17.04/moses/scripts/analysis/smtgui/\n",
            "ubuntu-17.04/moses/scripts/analysis/smtgui/newsmtgui.cgi\n",
            "ubuntu-17.04/moses/scripts/analysis/smtgui/file-factors\n",
            "ubuntu-17.04/moses/scripts/analysis/smtgui/Corpus.pm\n",
            "ubuntu-17.04/moses/scripts/analysis/smtgui/file-descriptions\n",
            "ubuntu-17.04/moses/scripts/analysis/smtgui/README\n",
            "ubuntu-17.04/moses/scripts/analysis/smtgui/filter-phrase-table.pl\n",
            "ubuntu-17.04/moses/scripts/analysis/bootstrap-hypothesis-difference-significance.pl\n",
            "ubuntu-17.04/moses/scripts/analysis/README\n",
            "ubuntu-17.04/moses/scripts/analysis/extract-target-trees.py\n",
            "ubuntu-17.04/moses/scripts/analysis/sentence-by-sentence.pl\n",
            "ubuntu-17.04/moses/scripts/analysis/show-phrases-used.pl\n",
            "ubuntu-17.04/moses/scripts/analysis/oov.pl\n",
            "ubuntu-17.04/moses/scripts/analysis/perllib/\n",
            "ubuntu-17.04/moses/scripts/analysis/perllib/Error.pm\n",
            "ubuntu-17.04/moses/scripts/analysis/suspicious_tokenization.pl\n",
            "ubuntu-17.04/moses/scripts/analysis/weight-scan.pl\n",
            "ubuntu-17.04/moses/scripts/analysis/nontranslated_words.pl\n",
            "ubuntu-17.04/moses/scripts/analysis/weight-scan-summarize.sh\n",
            "ubuntu-17.04/moses/scripts/generic/\n",
            "ubuntu-17.04/moses/scripts/generic/mteval-v14.pl\n",
            "ubuntu-17.04/moses/scripts/generic/mteval-v12.pl\n",
            "ubuntu-17.04/moses/scripts/generic/giza-parallel.perl\n",
            "ubuntu-17.04/moses/scripts/generic/score-parallel.perl\n",
            "ubuntu-17.04/moses/scripts/generic/binarize4moses2.perl\n",
            "ubuntu-17.04/moses/scripts/generic/extract-parallel.perl\n",
            "ubuntu-17.04/moses/scripts/generic/qsub-wrapper.pl\n",
            "ubuntu-17.04/moses/scripts/generic/fsa2fsal.pl\n",
            "ubuntu-17.04/moses/scripts/generic/reverse-alignment.perl\n",
            "ubuntu-17.04/moses/scripts/generic/bsbleu.py\n",
            "ubuntu-17.04/moses/scripts/generic/trainlm-irst2.perl\n",
            "ubuntu-17.04/moses/scripts/generic/strip-xml.perl\n",
            "ubuntu-17.04/moses/scripts/generic/moses_sim_pe.py\n",
            "ubuntu-17.04/moses/scripts/generic/multi-bleu.perl\n",
            "ubuntu-17.04/moses/scripts/generic/generic-parallel.perl\n",
            "ubuntu-17.04/moses/scripts/generic/lopar2pos.pl\n",
            "ubuntu-17.04/moses/scripts/generic/moses-parallel.pl\n",
            "ubuntu-17.04/moses/scripts/generic/ph_numbers.perl\n",
            "ubuntu-17.04/moses/scripts/generic/mteval-v13a.pl\n",
            "ubuntu-17.04/moses/scripts/generic/multi_moses.py\n",
            "ubuntu-17.04/moses/scripts/generic/extract-factors.pl\n",
            "ubuntu-17.04/moses/scripts/generic/fsa2plf.pl\n",
            "ubuntu-17.04/moses/scripts/generic/fsa-sample.fsa\n",
            "ubuntu-17.04/moses/scripts/generic/score_parallel.py\n",
            "ubuntu-17.04/moses/scripts/generic/fsal2fsa.pl\n",
            "ubuntu-17.04/moses/scripts/generic/compound-splitter.perl\n",
            "ubuntu-17.04/moses/scripts/generic/mteval-v11b.pl\n",
            "ubuntu-17.04/moses/scripts/training/\n",
            "ubuntu-17.04/moses/scripts/training/wrap_moses.py\n",
            "ubuntu-17.04/moses/scripts/training/build-generation-table.perl\n",
            "ubuntu-17.04/moses/scripts/training/train-global-lexicon-model.perl\n",
            "ubuntu-17.04/moses/scripts/training/remove-orphan-phrase-pairs-from-reordering-table.perl\n",
            "ubuntu-17.04/moses/scripts/training/absolutize_moses_model.pl\n",
            "ubuntu-17.04/moses/scripts/training/LexicalTranslationModel.pm\n",
            "ubuntu-17.04/moses/scripts/training/reduce-factors.perl\n",
            "ubuntu-17.04/moses/scripts/training/combine_factors_syntax.pl\n",
            "ubuntu-17.04/moses/scripts/training/threshold-filter.perl\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/test_nplm.py\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/README\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/extract_test.py\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/train_nplm.py\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/create_blm_ini.py\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/averageNullEmbedding.py\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/tag.sh\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/extract_training.py\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/reduce_ngrams.py\n",
            "ubuntu-17.04/moses/scripts/training/bilingual-lm/extract.py\n",
            "ubuntu-17.04/moses/scripts/training/Jamfile\n",
            "ubuntu-17.04/moses/scripts/training/train-neurallm.py\n",
            "ubuntu-17.04/moses/scripts/training/filter-rule-table.py\n",
            "ubuntu-17.04/moses/scripts/training/postprocess-lopar.perl\n",
            "ubuntu-17.04/moses/scripts/training/clone_moses_model.pl\n",
            "ubuntu-17.04/moses/scripts/training/clean-corpus-n-ratio.perl\n",
            "ubuntu-17.04/moses/scripts/training/get-lexical.perl\n",
            "ubuntu-17.04/moses/scripts/training/convert-moses-ini-to-v2.perl\n",
            "ubuntu-17.04/moses/scripts/training/reduce_combine.pl\n",
            "ubuntu-17.04/moses/scripts/training/create_nplm_ini.py\n",
            "ubuntu-17.04/moses/scripts/training/giza2bal.pl\n",
            "ubuntu-17.04/moses/scripts/training/strip-xml.perl\n",
            "ubuntu-17.04/moses/scripts/training/analyse_moses_model.pl\n",
            "ubuntu-17.04/moses/scripts/training/build-mmsapt.perl\n",
            "ubuntu-17.04/moses/scripts/training/flexibility_score.py\n",
            "ubuntu-17.04/moses/scripts/training/combine_factors.pl\n",
            "ubuntu-17.04/moses/scripts/training/reduce-topt-count.pl\n",
            "ubuntu-17.04/moses/scripts/training/train-model.perl\n",
            "ubuntu-17.04/moses/scripts/training/rdlm/\n",
            "ubuntu-17.04/moses/scripts/training/rdlm/extract_syntactic_ngrams.py\n",
            "ubuntu-17.04/moses/scripts/training/rdlm/README\n",
            "ubuntu-17.04/moses/scripts/training/rdlm/average_null_embedding.py\n",
            "ubuntu-17.04/moses/scripts/training/rdlm/train_rdlm.py\n",
            "ubuntu-17.04/moses/scripts/training/rdlm/extract_vocab.py\n",
            "ubuntu-17.04/moses/scripts/training/clean-corpus-n.perl\n",
            "ubuntu-17.04/moses/scripts/training/create_count_tables.py\n",
            "ubuntu-17.04/moses/scripts/training/exodus.perl\n",
            "ubuntu-17.04/moses/scripts/training/mert-moses.pl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-de-pos.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/conll2mosesxml.py\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-en-pos.mxpost.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/parse-en-senna.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/syntax-hyphen-splitting.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-suffix.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/find-unparseable.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/berkeleyparsed2mosesxml.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/mosesxml2berkeleyparsed.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/madamira-tok.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/adam-suffix-array/\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/adam-suffix-array/suffix-array-create.sh\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/adam-suffix-array/suffix-array-extract.sh\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/parse-de-berkeley.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-stem.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/parse-en-egret.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/berkeleyparsed2mosesxml_PTB.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/parse-en-collins.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/morfessor-wrapper.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/senna2brackets.py\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-de-morph.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/parse-de-bitpar.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/tree-converter-mosesxml.sh\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-en-porter.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/madamira-wrapper.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-pos.tree-tagger.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/parse-en-bllip.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/filter-excluded-lines.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-de-lemma.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/make-factor-brown-cluster-mkcls.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/parse-en-stanford.py\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/mada-wrapper.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/tagger-german-chunk.perl\n",
            "ubuntu-17.04/moses/scripts/training/wrappers/mosesxml2brackets.py\n",
            "ubuntu-17.04/moses/scripts/training/corpus-sizes.perl\n",
            "ubuntu-17.04/moses/scripts/training/binarize-model.perl\n",
            "ubuntu-17.04/moses/scripts/training/filter-model-given-input.pl\n",
            "ubuntu-17.04/moses/scripts/share/\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.en\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fr\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ro\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pt\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.de\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/README.txt\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sk\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.hu\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.zh\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.el\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ga\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.cs\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lt\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.nl\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.lv\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.es\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sl\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.is\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.it\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.yue\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ta\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.fi\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ru\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.sv\n",
            "ubuntu-17.04/moses/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.ca\n",
            "ubuntu-17.04/moses/scripts/Jamfile\n",
            "ubuntu-17.04/moses/scripts/tests/\n",
            "ubuntu-17.04/moses/scripts/tests/train-factored-test-step9.test\n",
            "ubuntu-17.04/moses/scripts/tests/README\n",
            "ubuntu-17.04/moses/scripts/tests/full-train-mert-decode.test\n",
            "ubuntu-17.04/moses/scripts/tests/cs-en-sample/\n",
            "ubuntu-17.04/moses/scripts/tests/cs-en-sample/train.cs\n",
            "ubuntu-17.04/moses/scripts/tests/cs-en-sample/test.src\n",
            "ubuntu-17.04/moses/scripts/tests/cs-en-sample/lm.en.gz\n",
            "ubuntu-17.04/moses/scripts/tests/cs-en-sample/train.en\n",
            "ubuntu-17.04/moses/scripts/tests/cs-en-sample/test.ref\n",
            "ubuntu-17.04/moses/scripts/OSM/\n",
            "ubuntu-17.04/moses/scripts/OSM/OSM-Train.perl\n",
            "ubuntu-17.04/moses/scripts/OSM/extract-singletons.perl\n",
            "ubuntu-17.04/moses/scripts/OSM/flipAlignment.perl\n",
            "ubuntu-17.04/moses/scripts/README\n",
            "ubuntu-17.04/moses/scripts/tokenizer/\n",
            "ubuntu-17.04/moses/scripts/tokenizer/deescape-special-chars-PTB.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/tokenizer.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/basic-protected-patterns\n",
            "ubuntu-17.04/moses/scripts/tokenizer/lowercase.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/tokenizer_PTB.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/replace-unicode-punctuation.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/normalize-punctuation.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/deescape-special-chars.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/pre-tokenizer.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/escape-special-chars.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/remove-non-printing-char.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/detokenizer.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/pre-tok-clean.perl\n",
            "ubuntu-17.04/moses/scripts/tokenizer/pre_tokenize_cleaning.py\n",
            "ubuntu-17.04/moses/scripts/tokenizer/delete-long-words.perl\n",
            "ubuntu-17.04/moses/scripts/ems/\n",
            "ubuntu-17.04/moses/scripts/ems/experiment.machines\n",
            "ubuntu-17.04/moses/scripts/ems/experiment.perl\n",
            "ubuntu-17.04/moses/scripts/ems/experiment.meta\n",
            "ubuntu-17.04/moses/scripts/ems/example/\n",
            "ubuntu-17.04/moses/scripts/ems/example/config.toy\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/nc-5k.en\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/weight.ini\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/nc-5k.fr\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/test-ref.en.sgm\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/weight_bilinguallm.ini\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/run.hierarchical.sh\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/test-src.fr.sgm\n",
            "ubuntu-17.04/moses/scripts/ems/example/data/config.hierarchical\n",
            "ubuntu-17.04/moses/scripts/ems/example/config.syntax\n",
            "ubuntu-17.04/moses/scripts/ems/example/config.basic\n",
            "ubuntu-17.04/moses/scripts/ems/example/config.factored\n",
            "ubuntu-17.04/moses/scripts/ems/example/config.toy.bilinguallm\n",
            "ubuntu-17.04/moses/scripts/ems/example/config.hierarchical\n",
            "ubuntu-17.04/moses/scripts/ems/support/\n",
            "ubuntu-17.04/moses/scripts/ems/support/ter.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/symmetrize-fast-align.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/submit-grid.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/interpolate-lm.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/tree-converter-wrapper.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/mml-filter.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/prepare-fast-align.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/consolidate-training-data.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/report-experiment-scores.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/split-sentences.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/run-wade.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/lmplz-wrapper.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/create-xml.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/generic-parallelizer.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/mml-filter.py\n",
            "ubuntu-17.04/moses/scripts/ems/support/wrap-xml.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/substitute-filtered-tables.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/build-domain-file-from-subcorpora.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/berkeley-train.sh\n",
            "ubuntu-17.04/moses/scripts/ems/support/substitute-filtered-tables-and-weights.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/analysis.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/berkeley-process.sh\n",
            "ubuntu-17.04/moses/scripts/ems/support/build-sparse-features.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/reference-from-sgm.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/defaultconfig.py\n",
            "ubuntu-17.04/moses/scripts/ems/support/mml-train.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/thot-lm-wrapper.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/remove-segmentation-markup.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/run-command-on-multiple-refsets.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/generic-multicore-parallelizer.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/input-from-sgm.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/fast-align-in-parts.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/substitute-weights.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/cache-model.perl\n",
            "ubuntu-17.04/moses/scripts/ems/support/mml-score.perl\n",
            "ubuntu-17.04/moses/scripts/ems/web/\n",
            "ubuntu-17.04/moses/scripts/ems/web/analysis.php\n",
            "ubuntu-17.04/moses/scripts/ems/web/close.gif\n",
            "ubuntu-17.04/moses/scripts/ems/web/hierarchical-segmentation.css\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/controls.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/builder.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/effects.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/slider.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/dragdrop.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/sound.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/prototype.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/controls.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/builder.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/effects.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/slider.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/scriptaculous.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/dragdrop.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/sound.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/src/unittest.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/run_functional_tests.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_grow_strink_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop_delay_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/icon.png\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effect_shake.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/ajax_autocompleter_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_queue_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/ajax_inplacecollectioneditor_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sortable_tree_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop3_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sword.mp3\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sortable_nested_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/_ajax_inplaceeditor_result.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_float_appear_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects6_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effect_scale_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/ajax_autocompleter2_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop6_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop9_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sortable_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sortable4_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/_autocomplete_result_single.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_toggle_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_random_demo.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects5b_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effect_direct_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sound_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_highlight_bg_image.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dropmarker.png\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sortable6_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects4_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_queue_limit_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/_autocomplete_result.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/texteffects_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/slider_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sortable2_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/index.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop7_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/_ajax_inplaceeditor_text.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/ajax_inplaceeditor_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effect_puff_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop8_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sortable3_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/position_clone_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects2_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects_blind_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects5_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop5_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop2_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/dragdrop4_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/sortable5_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/functional/effects3_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/test.css\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/run_unit_tests.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/icon.png\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/ajax_autocompleter_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/bdd_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/_ajax_inplaceeditor_result.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/_ajax_inplaceeditor_ipce_alt_text.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/element_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/_ajax_updater_result.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/sortable_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/_ajax_inplaceeditor_result2.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/_autocomplete_result.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/slider_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/_autocomplete_result_nobr.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/index.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/_ajax_inplaceeditor_text.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/ajax_inplaceeditor_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/string_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/_ajax_inplaceeditor_tagged.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/position_clone_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/effects_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/unittest_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/dragdrop_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/builder_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/test/unit/loading_test.html\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/CHANGELOG\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/README.rdoc\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/MIT-LICENSE\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/lib/\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/scriptaculous-js-1.8.3/lib/prototype.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/javascripts/unittest.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/index.php\n",
            "ubuntu-17.04/moses/scripts/ems/web/spinner.gif\n",
            "ubuntu-17.04/moses/scripts/ems/web/bilingual-concordance.css\n",
            "ubuntu-17.04/moses/scripts/ems/web/favicon.ico\n",
            "ubuntu-17.04/moses/scripts/ems/web/comment.php\n",
            "ubuntu-17.04/moses/scripts/ems/web/progress.perl\n",
            "ubuntu-17.04/moses/scripts/ems/web/general.css\n",
            "ubuntu-17.04/moses/scripts/ems/web/diff.php\n",
            "ubuntu-17.04/moses/scripts/ems/web/base64.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/sgviz.php\n",
            "ubuntu-17.04/moses/scripts/ems/web/sgviz.js\n",
            "ubuntu-17.04/moses/scripts/ems/web/overview.php\n",
            "ubuntu-17.04/moses/scripts/ems/web/lib.php\n",
            "ubuntu-17.04/moses/scripts/ems/web/analysis_diff.php\n",
            "ubuntu-17.04/moses/scripts/ems/web/setup\n",
            "ubuntu-17.04/moses/scripts/ems/web/hierarchical-segmentation.js\n",
            "ubuntu-17.04/moses/scripts/ems/fix-info.perl\n",
            "ubuntu-17.04/moses/scripts/Transliteration/\n",
            "ubuntu-17.04/moses/scripts/Transliteration/threshold.pl\n",
            "ubuntu-17.04/moses/scripts/Transliteration/clean.pl\n",
            "ubuntu-17.04/moses/scripts/Transliteration/prepare-transliteration-phrase-table.pl\n",
            "ubuntu-17.04/moses/scripts/Transliteration/in-decoding-transliteration.pl\n",
            "ubuntu-17.04/moses/scripts/Transliteration/post-decoding-transliteration.pl\n",
            "ubuntu-17.04/moses/scripts/Transliteration/train-transliteration-module.pl\n",
            "ubuntu-17.04/moses/scripts/Transliteration/corpusCreator.pl\n",
            "ubuntu-17.04/moses/scripts/server/\n",
            "ubuntu-17.04/moses/scripts/server/sim-pe.py\n",
            "ubuntu-17.04/moses/scripts/server/moses.py\n",
            "ubuntu-17.04/moses/scripts/fuzzy-match/\n",
            "ubuntu-17.04/moses/scripts/fuzzy-match/.cproject\n",
            "ubuntu-17.04/moses/scripts/fuzzy-match/create_xml.perl\n",
            "ubuntu-17.04/moses/scripts/fuzzy-match/.project\n",
            "ubuntu-17.04/moses/scripts/fuzzy-match/compile.sh\n",
            "ubuntu-17.04/moses/scripts/other/\n",
            "ubuntu-17.04/moses/scripts/other/delete-scores.perl\n",
            "ubuntu-17.04/moses/scripts/other/translate_by_microsoft_bing.perl\n",
            "ubuntu-17.04/moses/scripts/other/beautify.py\n",
            "ubuntu-17.04/moses/scripts/other/get_many_translations_from_google.perl\n",
            "ubuntu-17.04/moses/scripts/other/retain-lines.perl\n",
            "ubuntu-17.04/moses/scripts/other/gacha_filter.py\n",
            "ubuntu-17.04/moses/scripts/other/convert-pt.perl\n",
            "ubuntu-17.04/moses/scripts/other/buckwalter.perl\n",
            "ubuntu-17.04/moses/scripts/other/blame-stat.sh\n",
            "ubuntu-17.04/moses/scripts/docker/\n",
            "ubuntu-17.04/moses/scripts/docker/Dockerfile.ubuntu.basic\n",
            "ubuntu-17.04/moses/scripts/docker/Dockerfile.ubuntu.fastlightpbmt\n",
            "ubuntu-17.04/moses/scripts/nbest-rescore/\n",
            "ubuntu-17.04/moses/scripts/nbest-rescore/topbest.py\n",
            "ubuntu-17.04/moses/scripts/nbest-rescore/README.md\n",
            "ubuntu-17.04/moses/scripts/nbest-rescore/train.py\n",
            "ubuntu-17.04/moses/scripts/nbest-rescore/rescore.py\n",
            "ubuntu-17.04/moses/scripts/recaser/\n",
            "ubuntu-17.04/moses/scripts/recaser/recase.perl\n",
            "ubuntu-17.04/moses/scripts/recaser/detruecase.perl\n",
            "ubuntu-17.04/moses/scripts/recaser/truecase.perl\n",
            "ubuntu-17.04/moses/scripts/recaser/train-truecaser.perl\n",
            "ubuntu-17.04/moses/scripts/recaser/train-recaser.perl\n",
            "ubuntu-17.04/moses/scripts/regression-testing/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/moses-scripts-reg-test-data-1.0/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/moses-scripts-reg-test-data-1.0/lm/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/moses-scripts-reg-test-data-1.0/lm/train.en.w5.msb.nops.blm\n",
            "ubuntu-17.04/moses/scripts/regression-testing/moses-scripts-reg-test-data-1.0/models/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/moses-scripts-reg-test-data-1.0/models/phrase-table.0-0.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/moses-scripts-reg-test-data-1.0/models/reordering-table.msd-bidirectional-fe.0.5.0-0.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run2.features.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run3.features.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run5.features.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run1.scores.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run7.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run5.scores.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run5.names.txt\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run4.scores.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run6.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run1.features.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run7.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/finished_step.txt\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run4.features.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run3.scores.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run6.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/actual.index\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/archive.list\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run5.mert.log\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run5.weights.txt\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/data/run2.scores.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/moses.ini\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/filter-stderr\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/input\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/reference.2\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/filter-stdout\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/reference.1\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/truth/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/truth/results.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/reference.0\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-continue/command\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run4.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run3.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run5.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run4.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run5.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run2.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run1.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run2.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run1.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/run3.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/data/archive.list\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/moses.ini\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stderr\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/input\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/reference.2\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stdout\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/reference.1\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/truth/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/truth/results.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/reference.0\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-nocase/command\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run4.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run3.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run5.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run4.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run7.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run5.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run6.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run2.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run7.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run1.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run6.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run2.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run1.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/run3.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/data/archive.list\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/moses.ini\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/filter-stderr\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/input\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/reference.2\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/filter-stdout\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/reference.1\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/truth/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/truth/results.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/reference.0\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new/command\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run12.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run4.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run3.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run9.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run5.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run10.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run12.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run4.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run14.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run9.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run7.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run8.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run5.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run10.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run13.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run11.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run11.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run6.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run14.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run2.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run7.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run8.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run1.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run6.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run2.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run1.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run13.NBEST.out.gz\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/run3.TRANS.out\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/data/archive.list\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/moses.ini\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stderr\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/input\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/reference.2\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stdout\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/reference.1\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/truth/\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/truth/results.dat\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/reference.0\n",
            "ubuntu-17.04/moses/scripts/regression-testing/tests/mert-moses-new-aggregate/command\n",
            "ubuntu-17.04/moses/scripts/regression-testing/moses-virtual\n",
            "ubuntu-17.04/moses/scripts/regression-testing/MosesScriptsRegressionTesting.pm\n",
            "ubuntu-17.04/moses/scripts/regression-testing/run-single-test.pl\n",
            "ubuntu-17.04/moses/scripts/regression-testing/run-test-suite.pl\n",
            "ubuntu-17.04/moses/scripts/regression-testing/create_localized_moses_ini.pl\n",
            "ubuntu-17.04/moses/scripts/regression-testing/moses-virtual.pl\n",
            "ubuntu-17.04/moses/scripts/regression-testing/compare-results.pl\n",
            "ubuntu-17.04/moses/scripts/regression-testing/modify-pars.pl\n",
            "ubuntu-17.04/moses/bin/\n",
            "ubuntu-17.04/moses/bin/queryPhraseTableMin\n",
            "ubuntu-17.04/moses/bin/build_binary\n",
            "ubuntu-17.04/moses/bin/lexical-reordering-score\n",
            "ubuntu-17.04/moses/bin/relax-parse\n",
            "ubuntu-17.04/moses/bin/biconcor\n",
            "ubuntu-17.04/moses/bin/extract-lex\n",
            "ubuntu-17.04/moses/bin/symal2mam\n",
            "ubuntu-17.04/moses/bin/1-1-Extraction\n",
            "ubuntu-17.04/moses/bin/moses2\n",
            "ubuntu-17.04/moses/bin/filter\n",
            "ubuntu-17.04/moses/bin/lmbrgrid\n",
            "ubuntu-17.04/moses/bin/merge-sorted\n",
            "ubuntu-17.04/moses/bin/pcfg-score\n",
            "ubuntu-17.04/moses/bin/queryOnDiskPt\n",
            "ubuntu-17.04/moses/bin/project-cache.jam\n",
            "ubuntu-17.04/moses/bin/score-stsg\n",
            "ubuntu-17.04/moses/bin/addLexROtoPT\n",
            "ubuntu-17.04/moses/bin/CreateProbingPT2\n",
            "ubuntu-17.04/moses/bin/generateSequences\n",
            "ubuntu-17.04/moses/bin/vwtrainer\n",
            "ubuntu-17.04/moses/bin/statistics\n",
            "ubuntu-17.04/moses/bin/consolidate-reverse\n",
            "ubuntu-17.04/moses/bin/score\n",
            "ubuntu-17.04/moses/bin/extract-mixed-syntax\n",
            "ubuntu-17.04/moses/bin/mert\n",
            "ubuntu-17.04/moses/bin/query\n",
            "ubuntu-17.04/moses/bin/phrase_table_vocab\n",
            "ubuntu-17.04/moses/bin/config.log\n",
            "ubuntu-17.04/moses/bin/hgdecode\n",
            "ubuntu-17.04/moses/bin/mtt-build\n",
            "ubuntu-17.04/moses/bin/kenlm_benchmark\n",
            "ubuntu-17.04/moses/bin/consolidate-direct\n",
            "ubuntu-17.04/moses/bin/CreateProbingPT\n",
            "ubuntu-17.04/moses/bin/extract-ghkm\n",
            "ubuntu-17.04/moses/bin/pro\n",
            "ubuntu-17.04/moses/bin/dump_counts\n",
            "ubuntu-17.04/moses/bin/pcfg-extract\n",
            "ubuntu-17.04/moses/bin/pruneGeneration\n",
            "ubuntu-17.04/moses/bin/phrase-lookup\n",
            "ubuntu-17.04/moses/bin/evaluator\n",
            "ubuntu-17.04/moses/bin/prunePhraseTable\n",
            "ubuntu-17.04/moses/bin/kbmira\n",
            "ubuntu-17.04/moses/bin/sentence-bleu\n",
            "ubuntu-17.04/moses/bin/extract-rules\n",
            "ubuntu-17.04/moses/bin/filter-rule-table\n",
            "ubuntu-17.04/moses/bin/fragment\n",
            "ubuntu-17.04/moses/bin/moses\n",
            "ubuntu-17.04/moses/bin/moses_chart\n",
            "ubuntu-17.04/moses/bin/gcc-6.3.0/\n",
            "ubuntu-17.04/moses/bin/gcc-6.3.0/debug/\n",
            "ubuntu-17.04/moses/bin/gcc-6.3.0/debug/empty_test_shared.o\n",
            "ubuntu-17.04/moses/bin/gcc-6.3.0/debug/empty_test_static\n",
            "ubuntu-17.04/moses/bin/gcc-6.3.0/debug/empty_test_static.o\n",
            "ubuntu-17.04/moses/bin/gcc-6.3.0/debug/empty_test_shared\n",
            "ubuntu-17.04/moses/bin/processLexicalTableMin\n",
            "ubuntu-17.04/moses/bin/queryLexicalTable\n",
            "ubuntu-17.04/moses/bin/CreateOnDiskPt\n",
            "ubuntu-17.04/moses/bin/sentence-bleu-nbest\n",
            "ubuntu-17.04/moses/bin/processPhraseTableMin\n",
            "ubuntu-17.04/moses/bin/lmplz\n",
            "ubuntu-17.04/moses/bin/symal\n",
            "ubuntu-17.04/moses/bin/postprocess-egret-forests\n",
            "ubuntu-17.04/moses/bin/extract\n",
            "ubuntu-17.04/moses/bin/consolidate\n",
            "ubuntu-17.04/moses/bin/extractor\n",
            "ubuntu-17.04/moses/bin/mmlex-build\n",
            "ubuntu-17.04/moses/bin/mosesserver\n",
            "ubuntu-17.04/moses/bin/train-expected-bleu\n",
            "ubuntu-17.04/moses/bin/TMining\n",
            "ubuntu-17.04/moses/bin/processLexicalTable\n",
            "ubuntu-17.04/moses/bin/prepare-expected-bleu-training\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EPQ7Bpcln6x9"
      },
      "source": [
        "This is the repo which contains the parallel data we are going to use."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCYNy9nyAOHP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "054075d8-c013-42ab-ea63-8f2a8c725c60"
      },
      "source": [
        "# Cloning our repo\n",
        "%cd /content\n",
        "!git clone https://github.com/priyanshu2103/Sanskrit-Hindi-Machine-Translation.git"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "Cloning into 'Sanskrit-Hindi-Machine-Translation'...\n",
            "remote: Enumerating objects: 112, done.\u001b[K\n",
            "remote: Counting objects: 100% (112/112), done.\u001b[K\n",
            "remote: Compressing objects: 100% (97/97), done.\u001b[K\n",
            "remote: Total 112 (delta 38), reused 58 (delta 12), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (112/112), 9.25 MiB | 17.31 MiB/s, done.\n",
            "Resolving deltas: 100% (38/38), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OtVO52SeoX6H"
      },
      "source": [
        "We are using indic-nlp library for sanskrit text tokenization."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ciaAsTbZ__TX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f0f4f34f-7eeb-4bdb-89d5-7e670f58642d"
      },
      "source": [
        "# Install indic-nlp for tokenization\n",
        "!pip install indic-nlp-library\n",
        "!cp /content/Sanskrit-Hindi-Machine-Translation/indic_tokenize.py /usr/local/lib/python3.6/dist-packages/indicnlp/tokenize/indic_tokenize.py"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting indic-nlp-library\n",
            "  Downloading https://files.pythonhosted.org/packages/2f/51/f4e4542a226055b73a621ad442c16ae2c913d6b497283c99cae7a9661e6c/indic_nlp_library-0.71-py3-none-any.whl\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from indic-nlp-library) (1.1.4)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from indic-nlp-library) (1.18.5)\n",
            "Collecting morfessor\n",
            "  Downloading https://files.pythonhosted.org/packages/39/e6/7afea30be2ee4d29ce9de0fa53acbb033163615f849515c0b1956ad074ee/Morfessor-2.0.6-py3-none-any.whl\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas->indic-nlp-library) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->indic-nlp-library) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.7.3->pandas->indic-nlp-library) (1.15.0)\n",
            "Installing collected packages: morfessor, indic-nlp-library\n",
            "Successfully installed indic-nlp-library-0.71 morfessor-2.0.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_vINDHRO2xdg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "38708b38-7436-4f77-cd32-2b49f867a09a"
      },
      "source": [
        "# Extracting the parallel data into lists\n",
        "%cd /content/Sanskrit-Hindi-Machine-Translation/parallel-corpus/sanskrit-english/\n",
        "\n",
        "eng_lines = []\n",
        "sanskrit_lines = []\n",
        "\n",
        "with open('bhagvadgita_english.txt', 'r') as f:\n",
        "  eng_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "with open('bible_english.txt', 'r') as f:\n",
        "  eng_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "with open('manu_english.txt', 'r') as f:\n",
        "  eng_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "with open('ramayan_english.txt', 'r') as f:\n",
        "  eng_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "with open('rigveda_english.txt', 'r') as f:\n",
        "  eng_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "\n",
        "\n",
        "with open('bhagvadgita_sanskrit.txt', 'r') as f:\n",
        "  sanskrit_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "with open('bible_sanskrit.txt', 'r') as f:\n",
        "  sanskrit_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "with open('manu_sanskrit.txt', 'r') as f:\n",
        "  sanskrit_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "with open('ramayan_sanskrit.txt', 'r') as f:\n",
        "  sanskrit_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "with open('rigveda_sanskrit.txt', 'r') as f:\n",
        "  sanskrit_lines.extend([x.replace('\\n', '') for x in f.readlines()])\n",
        "\n",
        "# print(eng_lines[:100])\n",
        "# print(sanskrit_lines[:100])\n",
        "\n",
        "print(len(eng_lines))\n",
        "print(len(sanskrit_lines))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/Sanskrit-Hindi-Machine-Translation/parallel-corpus/sanskrit-english\n",
            "34374\n",
            "34374\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vISkiQdApCog"
      },
      "source": [
        "# Randomly shuffling the data into training and dev set, the test data is already provided on github\n",
        "import random\n",
        "c = list(zip(sanskrit_lines, eng_lines))\n",
        "random.shuffle(c)\n",
        "\n",
        "sanskrit_lines, eng_lines = zip(*c)\n",
        "\n",
        "train_text_sa = sanskrit_lines[:-1374]\n",
        "train_text_en = eng_lines[:-1374]\n",
        "\n",
        "dev_text_sa = sanskrit_lines[-1374:]\n",
        "dev_text_en = eng_lines[-1374:]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M8oFbearpX4j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03fa9dc3-da59-465e-c91a-ad8577beb886"
      },
      "source": [
        "# Setting up the required files\n",
        "%cd /content\n",
        "!mkdir -p corpus\n",
        "%cd corpus\n",
        "!mkdir -p training\n",
        "%cd training\n",
        "\n",
        "with open('parallel.sa-en.sa', 'w') as f:\n",
        "  for line in train_text_sa:\n",
        "    f.write(line + '\\n')\n",
        "\n",
        "with open('parallel.sa-en.en', 'w') as f:\n",
        "  for line in train_text_en:\n",
        "    f.write(line + '\\n')\n",
        "\n",
        "!mkdir -p /content/corpus/dev\n",
        "%cd /content/corpus/dev\n",
        "with open('dev.sa-en.sa', 'w') as f:\n",
        "  for line in dev_text_sa:\n",
        "    f.write(line + '\\n')\n",
        "\n",
        "with open('dev.sa-en.en', 'w') as f:\n",
        "  for line in dev_text_en:\n",
        "    f.write(line + '\\n')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "/content/corpus\n",
            "/content/corpus/training\n",
            "/content/corpus/dev\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cg6LQPb2349I",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2a600224-1699-492a-cf74-077f6a56bcf6"
      },
      "source": [
        "%cd /content"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k5XhckbTScbz"
      },
      "source": [
        "Tokenize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y6AGdiPn5_Vc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e072ea7b-461b-4c4f-bbd1-4dbc5dce21c7"
      },
      "source": [
        "!ubuntu-17.04/moses/scripts/tokenizer/tokenizer.perl -l en \\\n",
        "    < /content/corpus/training/parallel.sa-en.en    \\\n",
        "    > /content/corpus/parallel.sa-en.tok.en\n",
        "\n",
        "!python /usr/local/lib/python3.6/dist-packages/indicnlp/tokenize/indic_tokenize.py /content/corpus/training/parallel.sa-en.sa /content/corpus/parallel.sa-en.tok.sa sa"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tokenizer Version 1.1\n",
            "Language: en\n",
            "Number of threads: 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iIkYWJ62SeJl"
      },
      "source": [
        "Truecase\n",
        "\n",
        "We first need to learn the truecaser, then apply it"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ny_AvMqJ6KJm"
      },
      "source": [
        "!ubuntu-17.04/moses/scripts/recaser/train-truecaser.perl \\\n",
        "     --model /content/corpus/truecase-model.en --corpus     \\\n",
        "     /content/corpus/parallel.sa-en.tok.en\n",
        "!ubuntu-17.04/moses/scripts/recaser/train-truecaser.perl \\\n",
        "     --model /content/corpus/truecase-model.sa --corpus     \\\n",
        "     /content/corpus/parallel.sa-en.tok.sa"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "92YuccWo64li"
      },
      "source": [
        "!ubuntu-17.04/moses/scripts/recaser/truecase.perl \\\n",
        "   --model /content/corpus/truecase-model.en         \\\n",
        "   < /content/corpus/parallel.sa-en.tok.en \\\n",
        "   > /content/corpus/parallel.sa-en.true.en\n",
        "!ubuntu-17.04/moses/scripts/recaser/truecase.perl \\\n",
        "   --model /content/corpus/truecase-model.sa  \\\n",
        "   < /content/corpus/parallel.sa-en.tok.sa \\\n",
        "   > /content/corpus/parallel.sa-en.true.sa"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aU7xj0r8SpEk"
      },
      "source": [
        "Clean the data by filtering out sentences longer than 80 words and remove any blank lines"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlCJlOJU7pge",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "507af059-93a5-4717-c680-47ad943d1417"
      },
      "source": [
        "!/content/ubuntu-17.04/moses/scripts/training/clean-corpus-n.perl \\\n",
        "    /content/corpus/parallel.sa-en.true sa en \\\n",
        "    /content/corpus/parallel.sa-en.clean 1 80"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "clean-corpus.perl: processing /content/corpus/parallel.sa-en.true.sa & .en to /content/corpus/parallel.sa-en.clean, cutoff 1-80, ratio 9\n",
            "...\n",
            "Input sentences: 33000  Output sentences:  32621\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "andLXpdgSzYC"
      },
      "source": [
        "Train the Language Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yk23m2Rr73l6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "46637a3e-14b4-4f73-9601-f2152b5ac84b"
      },
      "source": [
        "!mkdir -p /content/lm\n",
        "%cd /content/lm\n",
        "!/content/ubuntu-17.04/moses/bin/lmplz -o 3 < /content/corpus/parallel.sa-en.true.en > parallel.sa-en.arpa.en"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/lm\n",
            "=== 1/5 Counting and sorting n-grams ===\n",
            "Reading /content/corpus/parallel.sa-en.true.en\n",
            "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
            "tcmalloc: large alloc 2514862080 bytes == 0x55adabb84000 @  0x7fa5756211e7 0x55adaade33f4 0x55adaae5f900 0x55adaae4e800 0x55adaada1110 0x7fa5744b8bf7 0x55adaada2aca\n",
            "tcmalloc: large alloc 8382865408 bytes == 0x55ae419e0000 @  0x7fa5756211e7 0x55adaade33f4 0x55adaae3aa3e 0x55adaae3b3fe 0x55adaae4e817 0x55adaada1110 0x7fa5744b8bf7 0x55adaada2aca\n",
            "****************************************************************************************************\n",
            "Unigram tokens 817144 types 24559\n",
            "=== 2/5 Calculating and sorting adjusted counts ===\n",
            "Chain sizes: 1:294708 2:3799160832 3:7123426816\n",
            "tcmalloc: large alloc 7123427328 bytes == 0x55adabb84000 @  0x7fa5756211e7 0x55adaade33f4 0x55adaae3aa3e 0x55adaae3b3fe 0x55adaae4ed5d 0x55adaada1110 0x7fa5744b8bf7 0x55adaada2aca\n",
            "tcmalloc: large alloc 3799162880 bytes == 0x55b035d12000 @  0x7fa5756211e7 0x55adaade33f4 0x55adaae3aa3e 0x55adaae3b3fe 0x55adaae4f195 0x55adaada1110 0x7fa5744b8bf7 0x55adaada2aca\n",
            "Statistics:\n",
            "1 24559 D1=0.641434 D2=0.974283 D3+=1.25241\n",
            "2 238390 D1=0.7445 D2=1.09687 D3+=1.42956\n",
            "3 533070 D1=0.833912 D2=1.18933 D3+=1.36432\n",
            "Memory estimate for binary LM:\n",
            "type       kB\n",
            "probing 15581 assuming -p 1.5\n",
            "probing 17073 assuming -r models -p 1.5\n",
            "trie     6420 without quantization\n",
            "trie     3559 assuming -q 8 -b 8 quantization \n",
            "trie     6075 assuming -a 22 array pointer compression\n",
            "trie     3213 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
            "=== 3/5 Calculating and sorting initial probabilities ===\n",
            "Chain sizes: 1:294708 2:3814240 3:10661400\n",
            "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
            "####################################################################################################\n",
            "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
            "Chain sizes: 1:294708 2:3814240 3:10661400\n",
            "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
            "####################################################################################################\n",
            "=== 5/5 Writing ARPA model ===\n",
            "Name:lmplz\tVmPeak:14603308 kB\tVmRSS:2506356 kB\tRSSMax:2508804 kB\tuser:1.22145\tsys:1.14281\tCPU:2.36426\treal:2.1897\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-5SWolBl8MvQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9550c346-709c-4765-ac83-f9df6112ea03"
      },
      "source": [
        "!/content/ubuntu-17.04/moses/bin/build_binary parallel.sa-en.arpa.en parallel.sa-en.blm.en"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading parallel.sa-en.arpa.en\n",
            "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
            "****************************************************************************************************\n",
            "SUCCESS\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ChwKtkY-8nDM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "934eaec7-bc87-4af5-f527-c6f883b79e3c"
      },
      "source": [
        "# Simple sanity check to see if correct process uptil now\n",
        "!echo \"is this an English sentence ?\"                       \\\n",
        "   | /content/ubuntu-17.04/moses/bin/query parallel.sa-en.blm.en"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "is=16 2 -2.9549935\tthis=6 2 -2.1058443\tan=110 2 -3.5131161\tEnglish=0 1 -5.8624034\tsentence=10004 1 -4.6240244\t?=56 1 -2.667434\t</s>=2 2 -0.37507707\tTotal: -22.102894 OOV: 1\n",
            "Perplexity including OOVs:\t1437.329242285302\n",
            "Perplexity excluding OOVs:\t509.03589279121127\n",
            "OOVs:\t1\n",
            "Tokens:\t7\n",
            "Name:query\tVmPeak:53840 kB\tVmRSS:6228 kB\tRSSMax:18236 kB\tuser:0.000998\tsys:0.002995\tCPU:0.003993\treal:0.00166383\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PxgaqXWZS_A8"
      },
      "source": [
        "## Actual training using mgiza alignment tool.\n",
        "### It will show that the cell has finished running instantly, however it is actually running in the background. Status can be seen at /working/training.out. Process is finished when the end of line in file says \"(9) create moses.ini\". Proceed further then"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sngxz8e09LPy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8941d809-21ff-41c8-c5e4-d2180fcb8626"
      },
      "source": [
        "!mkdir -p /content/working\n",
        "%cd /content/working\n",
        "!nohup nice /content/ubuntu-17.04/moses/scripts/training/train-model.perl -root-dir train \\\n",
        " -mgiza \\\n",
        " -corpus /content/corpus/parallel.sa-en.clean \\\n",
        " -f sa -e en -alignment grow-diag-final-and -reordering msd-bidirectional-fe \\\n",
        " -lm 0:3:/content/lm/parallel.sa-en.blm.en:8                          \\\n",
        " -cores 4 \\\n",
        " -external-bin-dir /content/ubuntu-17.04/training-tools >& training.out &"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/working\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K3k_DDpfQF12",
        "outputId": "39690ab7-3581-4189-b3af-cc263d0098f3"
      },
      "source": [
        "%cd /content/corpus\n",
        "!/content/ubuntu-17.04/moses/scripts/tokenizer/tokenizer.perl -l en \\\n",
        "   < dev/dev.sa-en.en > dev.sa-en.tok.en\n",
        "!python /usr/local/lib/python3.6/dist-packages/indicnlp/tokenize/indic_tokenize.py /content/corpus/dev/dev.sa-en.sa /content/corpus/dev.sa-en.tok.sa sa\n",
        "\n",
        "!/content/ubuntu-17.04/moses/scripts/recaser/truecase.perl --model truecase-model.en \\\n",
        "   < dev.sa-en.tok.en > dev.sa-en.true.en\n",
        "!/content/ubuntu-17.04/moses/scripts/recaser/truecase.perl --model truecase-model.sa \\\n",
        "   < dev.sa-en.tok.sa > dev.sa-en.true.sa"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/corpus\n",
            "Tokenizer Version 1.1\n",
            "Language: en\n",
            "Number of threads: 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ffLKfHeDi9q"
      },
      "source": [
        "## Tuning\n",
        "### It will show that the cell has finished running instantly, however it is actually running in the background. Status can be seen at /working/mert.out. Process is finished moses.ini file is formed inside /working/mert-work"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q4_5FRGRThMk",
        "outputId": "96aae2b5-d7c5-4022-b1a0-4b4c5b5cfb53"
      },
      "source": [
        "%cd /content/working\n",
        "!nohup nice /content/ubuntu-17.04/moses/scripts/training/mert-moses.pl \\\n",
        "  /content/corpus/dev.sa-en.true.sa /content/corpus/dev.sa-en.true.en \\\n",
        "  /content/ubuntu-17.04/moses/bin/moses train/model/moses.ini --mertdir /content/ubuntu-17.04/moses/bin/ \\\n",
        "  --decoder-flags=\"-threads 4\" \\\n",
        "  &> mert.out &"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/working\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7wHUDujAm8Ay",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3fb1b5c-9c5b-4788-9300-cbf5c6ab5e46"
      },
      "source": [
        "%cd /content/corpus\n",
        "# !/content/ubuntu-17.04/moses/scripts/tokenizer/tokenizer.perl -l en \\\n",
        "#    < test/dev.sa-en.en > dev.sa-en.tok.en\n",
        "!python /usr/local/lib/python3.6/dist-packages/indicnlp/tokenize/indic_tokenize.py /content/Sanskrit-Hindi-Machine-Translation/parallel-corpus/sanskrit-hindi/test_sanskrit.sa /content/corpus/test.tok.sa sa\n",
        "\n",
        "# !/content/ubuntu-17.04/moses/scripts/recaser/truecase.perl --model truecase-model.en \\\n",
        "#    < dev.sa-en.tok.en > dev.sa-en.true.en\n",
        "!/content/ubuntu-17.04/moses/scripts/recaser/truecase.perl --model truecase-model.sa \\\n",
        "   < test.tok.sa > test.true.sa"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/corpus\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jKlAbU602Vhc"
      },
      "source": [
        "!nohup nice /content/ubuntu-17.04/moses/bin/moses \\\n",
        "   -f /content/working/mert-work/moses.ini   \\\n",
        "   < /content/corpus/test.true.sa  \\\n",
        "   > /content/working/test.translated.en  \\\n",
        "   2> /content/working/test_result.out\n",
        "\n",
        "\n",
        "# !/content/ubuntu-17.04/moses/scripts/generic/multi-bleu.perl \\\n",
        "#    -lc /content/corpus/dev.sa-en.true.en    \\\n",
        "#    < /content/working/test.translated.en"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pHapMi2-UeOK"
      },
      "source": [
        "Google translate API to translate English to Hindi"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "my_dIFPEiDcw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fefb8683-44af-40d4-9d1a-d00207eb45a0"
      },
      "source": [
        "!pip install googletrans\n",
        "from googletrans import Translator\n",
        "translator = Translator()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting googletrans\n",
            "  Downloading https://files.pythonhosted.org/packages/71/3a/3b19effdd4c03958b90f40fe01c93de6d5280e03843cc5adf6956bfc9512/googletrans-3.0.0.tar.gz\n",
            "Collecting httpx==0.13.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/54/b4/698b284c6aed4d7c2b4fe3ba5df1fcf6093612423797e76fbb24890dd22f/httpx-0.13.3-py3-none-any.whl (55kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 5.0MB/s \n",
            "\u001b[?25hCollecting rfc3986<2,>=1.3\n",
            "  Downloading https://files.pythonhosted.org/packages/78/be/7b8b99fd74ff5684225f50dd0e865393d2265656ef3b4ba9eaaaffe622b8/rfc3986-1.4.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: chardet==3.* in /usr/local/lib/python3.6/dist-packages (from httpx==0.13.3->googletrans) (3.0.4)\n",
            "Collecting hstspreload\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d3/3c/cdeaf9ab0404853e77c45d9e8021d0d2c01f70a1bb26e460090926fe2a5e/hstspreload-2020.11.21-py3-none-any.whl (981kB)\n",
            "\u001b[K     |████████████████████████████████| 983kB 8.4MB/s \n",
            "\u001b[?25hCollecting httpcore==0.9.*\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/dd/d5/e4ff9318693ac6101a2095e580908b591838c6f33df8d3ee8dd953ba96a8/httpcore-0.9.1-py3-none-any.whl (42kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 8.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: idna==2.* in /usr/local/lib/python3.6/dist-packages (from httpx==0.13.3->googletrans) (2.10)\n",
            "Collecting sniffio\n",
            "  Downloading https://files.pythonhosted.org/packages/52/b0/7b2e028b63d092804b6794595871f936aafa5e9322dcaaad50ebf67445b3/sniffio-1.2.0-py3-none-any.whl\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from httpx==0.13.3->googletrans) (2020.6.20)\n",
            "Collecting h2==3.*\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/25/de/da019bcc539eeab02f6d45836f23858ac467f584bfec7a526ef200242afe/h2-3.2.0-py2.py3-none-any.whl (65kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 8.9MB/s \n",
            "\u001b[?25hCollecting h11<0.10,>=0.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5a/fd/3dad730b0f95e78aeeb742f96fa7bbecbdd56a58e405d3da440d5bfb90c6/h11-0.9.0-py2.py3-none-any.whl (53kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 8.5MB/s \n",
            "\u001b[?25hCollecting contextvars>=2.1; python_version < \"3.7\"\n",
            "  Downloading https://files.pythonhosted.org/packages/83/96/55b82d9f13763be9d672622e1b8106c85acb83edd7cc2fa5bc67cd9877e9/contextvars-2.4.tar.gz\n",
            "Collecting hyperframe<6,>=5.2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/19/0c/bf88182bcb5dce3094e2f3e4fe20db28a9928cb7bd5b08024030e4b140db/hyperframe-5.2.0-py2.py3-none-any.whl\n",
            "Collecting hpack<4,>=3.0\n",
            "  Downloading https://files.pythonhosted.org/packages/8a/cc/e53517f4a1e13f74776ca93271caef378dadec14d71c61c949d759d3db69/hpack-3.0.0-py2.py3-none-any.whl\n",
            "Collecting immutables>=0.9\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/e0/ea6fd4697120327d26773b5a84853f897a68e33d3f9376b00a8ff96e4f63/immutables-0.14-cp36-cp36m-manylinux1_x86_64.whl (98kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 10.5MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: googletrans, contextvars\n",
            "  Building wheel for googletrans (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for googletrans: filename=googletrans-3.0.0-cp36-none-any.whl size=15736 sha256=419af7f721163995aaefdbda22738dfffd091e69e8ac9842be9f2fba9737fb26\n",
            "  Stored in directory: /root/.cache/pip/wheels/28/1a/a7/eaf4d7a3417a0c65796c547cff4deb6d79c7d14c2abd29273e\n",
            "  Building wheel for contextvars (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for contextvars: filename=contextvars-2.4-cp36-none-any.whl size=7666 sha256=f5a13b0ed7085d686a7d743997e8efed9d1e3b90d25c66b8f8e910ae552cc81e\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/7d/68/1ebae2668bda2228686e3c1cf16f2c2384cea6e9334ad5f6de\n",
            "Successfully built googletrans contextvars\n",
            "Installing collected packages: rfc3986, hstspreload, immutables, contextvars, sniffio, hyperframe, hpack, h2, h11, httpcore, httpx, googletrans\n",
            "Successfully installed contextvars-2.4 googletrans-3.0.0 h11-0.9.0 h2-3.2.0 hpack-3.0.0 hstspreload-2020.11.21 httpcore-0.9.1 httpx-0.13.3 hyperframe-5.2.0 immutables-0.14 rfc3986-1.4.0 sniffio-1.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5LskrGNkNTbA"
      },
      "source": [
        "For translating multiple lines from file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VPEnERO93fpF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d0aea7d4-dbeb-4c23-d246-da01f22563f6"
      },
      "source": [
        "with open('/content/working/test.translated.en', 'r') as f:\n",
        "  test = [x.replace('\\n', '') for x in f.readlines()]\n",
        "\n",
        "translated = []\n",
        "print(len(test))\n",
        "\n",
        "for line in test:\n",
        "  ans = translator.translate(line, dest='hi')\n",
        "  translated.append(ans.text)\n",
        "\n",
        "with open('translated_hindi.hi', 'w') as f:\n",
        "  for line in translated:\n",
        "    f.write(line + '\\n')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1374\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pBbmEcc99UZQ",
        "outputId": "4d53b539-994a-4d8e-b5fb-7e8b4bd7e1de"
      },
      "source": [
        "# calculate bleu score\n",
        "import nltk\n",
        "with open('/content/Sanskrit-Hindi-Machine-Translation/parallel-corpus/sanskrit-hindi/test_hindi.hi', 'r') as f: # ground truth file\n",
        "  reference = [x.replace('\\n', '') for x in f.readlines()]\n",
        "\n",
        "with open('translated_hindi.hi', 'r') as f: # translated hindi file (our output)\n",
        "  candidate = [x.replace('\\n', '') for x in f.readlines()]\n",
        "\n",
        "from nltk.translate.bleu_score import SmoothingFunction\n",
        "smoothie = SmoothingFunction().method4\n",
        "\n",
        "print(\"bleu\", nltk.translate.bleu_score.corpus_bleu(reference, candidate, smoothing_function=smoothie))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "bleu 0.2830206410946437\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Aa0Qsvz5NeQ2"
      },
      "source": [
        "For translating single line present in /content/input.txt"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UBCtMXFlK2de"
      },
      "source": [
        "!/content/ubuntu-17.04/moses/bin/moses -f /content/working/mert-work/moses.ini < /content/input.txt > /content/temp.txt\n",
        "with open('/content/temp.txt', 'r') as f:\n",
        "  line = f.readline\n",
        "ans = translator.translate(line, dest='hi')\n",
        "print(ans.text)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}