project.yml

title: "la_core_web_xx"
description: "Code required to train spaCy-compatible sm, md, lg, and trf core models for Latin, i.e pipeline with POS tagger, morphologizer, lemmatizer, dependency parser, and NER trained on all available Latin UD treebanks, i.e. Perseus, PROIEL, ITTB, UDante, and LLCT (as currently normalized by Gamba/Zeman 2003; see `assets` below); the lemmatizer and tagger components are also trained on LASLA Corpus connlup files (see `assets` below).  The md and lg models contains floret vectors trained on Wikipedia, Oscar, UD and—for lg—CC100-Latin data. The trf model uses bert-base-multilingual-uncased. NER is based on custom tagged data based on tagger output and manual annotation, supplemented by data from the Herodotos Project; this data is included in `assets/ner/`."

vars:
  config: "config_lg"
  lang: "la"
  package_name: "core_web_"
  package_version: "3.7.7"
  gpu: 0
  treebank_pers: "UD_Latin-Perseus"
  treebank_proi: "UD_Latin-PROIEL"
  treebank_ittb: "UD_Latin-ITTB"
  treebank_llct: "UD_Latin-LLCT"
  treebank_udan: "UD_Latin-UDante"
  treebank_lila: "UD_Latin-CIRCSE"
  treebank_pers_data: "MM-la_perseus-ud"
  treebank_proi_data: "MM-la_proiel-ud"
  treebank_ittb_data: "MM-la_ittb-ud"
  treebank_llct_data: "MM-la_llct-ud"
  treebank_udan_data: "MM-la_udante-ud"
  treebank_lasl_data: "MM-la_lasla-xx"
  treebank_lila_data: "MM-la_lila-ud"

  # NER vars
  ud_ner_train: "ner/ud-ner-train.json"
  ud_ner_dev: "ner/ud-ner-dev.json"
  herodotos_ner_train: "ner/herodotos-ner-train.json"
  herodotos_ner_dev: "ner/herodotos-ner-train.json"

directories: ["assets", "corpus", "training", "configs", "metrics", "packages", "scripts"]

assets:
  - dest: "assets/original/ud"
    git:
      repo: "https://github.com/fjambe/Latin-variability"
      branch: "main"
      path: ""
  - dest: "assets/original/lasla"
    git:
      repo: "https://github.com/CIRCSE/LASLA"
      branch: "main"
      path: ""
  - dest: "assets/original/lila"
    git:
      repo: "https://github.com/UniversalDependencies/UD_Latin-CIRCSE"
      branch: "master"
      path: ""

workflows:
  all:
    # - assets 
    # - preprocess 
    # - convert 
    - init-labels
    - train_sm
    - train-lemma_sm
    - train_md
    - train-lemma_md    
    - train_lg   
    - train-lemma_lg    
    - train_trf     
    - train-lemma_trf    
    - evaluate_sm
    - evaluate_md
    - evaluate_lg    
    - evaluate_trf
    - convert-ner
    - train-ner_sm
    - train-ner_md
    - train-ner_lg
    - train-ner_trf        
    - assemble_sm
    - assemble_md
    - assemble_lg
    - assemble_trf
    - assemble-meta_sm
    - assemble-meta_md
    - assemble-meta_lg     
    - assemble-meta_trf
    - package_sm
    - package_md
    - package_lg       
    - package_trf 
    - document    
    # - clean

commands:
  - name: assets
    help: "Download assets"
    script:
      - "mkdir -p assets/original/ud"
      - "mkdir -p assets/original/lasla"
      - sh -c "rm -rf assets/original/ud/*"
      - sh -c "rm -rf assets/original/lasla/*" 
      - "mkdir -p assets/temp/ud"
      - "mkdir -p assets/temp/lasla"
      - python -m spacy project assets
      - sh -c "cp -r assets/original/ud/morpho_harmonization/morpho-harmonized-treebanks/ assets/temp/ud"
      - sh -c "rm -rf assets/original/ud/*"
      - sh -c "cp -r assets/temp/ud assets/original"
      - sh -c "rm -rf assets/temp/ud"
      - sh -c "cp -r assets/original/lasla/conllup/ assets/temp/lasla"
      - python scripts/prep_lasla.py
      - sh -c "rm -rf assets/original/lasla/*"
      - sh -c "cp -r assets/temp/lasla/lasla.conllu assets/original/lasla/MM-la_lasla-xx-train.conllu"
      - sh -c "rm -rf assets/temp/lasla"
      # temporarily rename lila to train until more data is available
      - sh -c "mv assets/original/lila/la_circse-ud-test.conllu assets/original/lila/MM-la_circse-ud-train.conllu"
    outputs:
      - "assets/original/ud/${vars.treebank_pers}"
      - "assets/original/ud/${vars.treebank_proi}"
      - "assets/original/ud/${vars.treebank_ittb}"
      - "assets/original/ud/${vars.treebank_udan}"
      - "assets/original/ud/${vars.treebank_llct}"
      - "assets/original/lasla/MM-la_lasla-xx-train.conllu"
      - "assets/original/lila/MM-la_circse-ud-train.conllu"
  - name: preprocess
    help: "Convert different UD treebanks so that they use shared formatting, feature defs, etc."
    script:
      - "mkdir -p assets/preprocess"
      - python scripts/update_conllu.py
      - python scripts/analyze_feats.py 
      - python scripts/remorph_lasla.py
    outputs:
      - "assets/preprocess/${vars.treebank_pers_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_pers_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_proi_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_proi_data}-dev.conllu"
      - "assets/preprocess/${vars.treebank_proi_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_ittb_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_ittb_data}-dev.conllu"
      - "assets/preprocess/${vars.treebank_ittb_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_llct_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_llct_data}-dev.conllu"
      - "assets/preprocess/${vars.treebank_llct_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_udan_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_udan_data}-dev.conllu"
      - "assets/preprocess/${vars.treebank_udan_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_lasl_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_lila_data}-train.conllu"      
  - name: convert
    help: "Convert the data to spaCy's format"
    script:
      - "mkdir -p corpus/train"
      - "mkdir -p corpus/dev"
      - "mkdir -p corpus/test"
      - "mkdir -p corpus/lemma/train"
      - "mkdir -p corpus/lemma/dev"
      - "mkdir -p corpus/lemma/test"
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_lasl_data}-train.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_pers_data}-train.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_pers_data}-test.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_proi_data}-train.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_proi_data}-dev.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_proi_data}-test.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_ittb_data}-train.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_ittb_data}-dev.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_ittb_data}-test.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_llct_data}-train.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_llct_data}-dev.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_llct_data}-test.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_udan_data}-train.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_udan_data}-dev.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - >-
        python -m spacy convert
        assets/preprocess/${vars.treebank_udan_data}-test.conllu 
        corpus/ 
        --converter conllu 
        --n-sents 10 
      - sh -c 'mv corpus/*lasla*.spacy corpus/lemma/train'
      - sh -c 'cp corpus/*train.spacy corpus/train/'
      - sh -c 'cp corpus/*dev.spacy corpus/dev/'
      - sh -c 'cp corpus/*test.spacy corpus/test/'
      - sh -c 'mv corpus/*train.spacy corpus/lemma/train/'
      - sh -c 'mv corpus/*dev.spacy corpus/lemma/dev/'
      - sh -c 'mv corpus/*test.spacy corpus/lemma/test/'

    deps:
      - "assets/preprocess/${vars.treebank_lasl_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_pers_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_pers_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_proi_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_proi_data}-dev.conllu"
      - "assets/preprocess/${vars.treebank_proi_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_ittb_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_ittb_data}-dev.conllu"
      - "assets/preprocess/${vars.treebank_ittb_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_llct_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_llct_data}-dev.conllu"
      - "assets/preprocess/${vars.treebank_llct_data}-test.conllu"
      - "assets/preprocess/${vars.treebank_udan_data}-train.conllu"
      - "assets/preprocess/${vars.treebank_udan_data}-dev.conllu"
      - "assets/preprocess/${vars.treebank_udan_data}-test.conllu"
    outputs:
      - "corpus/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_udan_data}-dev.spacy"
      - "corpus/test/${vars.treebank_pers_data}-test.spacy"
      - "corpus/test/${vars.treebank_proi_data}-test.spacy"
      - "corpus/test/${vars.treebank_ittb_data}-test.spacy"
      - "corpus/test/${vars.treebank_llct_data}-test.spacy"
      - "corpus/test/${vars.treebank_udan_data}-test.spacy"
      - "corpus/lemma/train/${vars.treebank_lasl_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/lemma/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_udan_data}-dev.spacy"
      - "corpus/lemma/test/${vars.treebank_pers_data}-test.spacy"
      - "corpus/lemma/test/${vars.treebank_proi_data}-test.spacy"
      - "corpus/lemma/test/${vars.treebank_ittb_data}-test.spacy"
      - "corpus/lemma/test/${vars.treebank_llct_data}-test.spacy"
      - "corpus/lemma/test/${vars.treebank_udan_data}-test.spacy"

  - name: init-labels
    help: "Initialize labels for components"
    script:
      - >-
        python -m spacy init labels 
        configs/config_sm.cfg
        corpus/labels
        --gpu-id ${vars.gpu} 
        --paths.train corpus/train
        --paths.dev corpus/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/train"
      - "corpus/dev"
      - "configs/config_sm.cfg"
    outputs:
      - "corpus/labels/morphologizer.json"
      - "corpus/labels/parser.json"
      - "corpus/labels/tagger.json"
      - "corpus/labels/trainable_lemmatizer.json"              

  - name: train_sm
    help: "Train tagger/parser/etc. on Latin UD treebanks"
    script:
      - "mkdir -p training/sm"
      - >-
        python -m spacy train 
        configs/config_sm.cfg
        --output training/sm
        --gpu-id ${vars.gpu} 
        --paths.train corpus/train
        --paths.dev corpus/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_udan_data}-dev.spacy"
      - "configs/config_sm.cfg"
    outputs:
      - "training/sm/model-best"

  - name: train-lemma_sm
    help: "Retrain tagger/parser/etc. on Latin UD treebanks and LASLA"
    script:
      - "mkdir -p training/sm/lemma"
      - >-
        python -m spacy train 
        configs/lemma_config_sm.cfg
        --output training/sm/lemma
        --gpu-id ${vars.gpu} 
        --paths.train corpus/lemma/train
        --paths.dev corpus/lemma/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/lemma/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_lasl_data}-train.spacy"
      - "corpus/lemma/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_udan_data}-dev.spacy"
      - "configs/lemma_config_sm.cfg"
    outputs:
      - "training/sm/lemma/model-best"

  - name: train_md
    help: "Train tagger/parser/etc. on Latin UD treebanks"
    script:
      - "mkdir -p training/md"
      - >-
        python -m spacy train 
        configs/config_md.cfg
        --output training/md
        --gpu-id ${vars.gpu} 
        --paths.train corpus/train
        --paths.dev corpus/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_udan_data}-dev.spacy"
      - "corpus/test/${vars.treebank_pers_data}-test.spacy"
      - "corpus/test/${vars.treebank_proi_data}-test.spacy"
      - "corpus/test/${vars.treebank_ittb_data}-test.spacy"
      - "corpus/test/${vars.treebank_llct_data}-test.spacy"
      - "corpus/test/${vars.treebank_udan_data}-test.spacy"    
      - "configs/config_md.cfg"
    outputs:
      - "training/md/model-best"      

  - name: train-lemma_md
    help: "Retrain tagger/parser/etc. on Latin UD treebanks and LASLA"
    script:
      - "mkdir -p training/md/lemma"
      - >-
        python -m spacy train 
        configs/lemma_config_md.cfg
        --output training/md/lemma
        --gpu-id ${vars.gpu} 
        --paths.train corpus/lemma/train
        --paths.dev corpus/lemma/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/lemma/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_lasl_data}-train.spacy"
      - "corpus/lemma/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_udan_data}-dev.spacy"
      - "configs/lemma_config_md.cfg"
    outputs:
      - "training/md/lemma/model-best"


  - name: train_lg
    help: "Train tagger/parser/etc. on Latin UD treebanks"
    script:
      - "mkdir -p training/lg"
      - >-
        python -m spacy train 
        configs/config_lg.cfg
        --output training/lg
        --gpu-id ${vars.gpu} 
        --paths.train corpus/train
        --paths.dev corpus/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_udan_data}-dev.spacy"
      - "corpus/test/${vars.treebank_pers_data}-test.spacy"
      - "corpus/test/${vars.treebank_proi_data}-test.spacy"
      - "corpus/test/${vars.treebank_ittb_data}-test.spacy"
      - "corpus/test/${vars.treebank_llct_data}-test.spacy"
      - "corpus/test/${vars.treebank_udan_data}-test.spacy"    
      - "configs/config_lg.cfg"
    outputs:
      - "training/lg/model-best"   

  - name: train-lemma_lg
    help: "Retrain tagger/parser/etc. on Latin UD treebanks and LASLA"
    script:
      - "mkdir -p training/lg/lemma"
      - >-
        python -m spacy train 
        configs/lemma_config_lg.cfg
        --output training/lg/lemma
        --gpu-id ${vars.gpu} 
        --paths.train corpus/lemma/train
        --paths.dev corpus/lemma/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/lemma/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_lasl_data}-train.spacy"
      - "corpus/lemma/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_udan_data}-dev.spacy"
      - "configs/lemma_config_lg.cfg"
    outputs:
      - "training/lg/lemma/model-best"


  - name: train_trf
    help: "Train tagger/parser/etc. on Latin UD treebanks"
    script:
      - >-
        python -m spacy train 
        configs/config_trf.cfg
        --output training/trf
        --gpu-id ${vars.gpu} 
        --paths.train corpus/train
        --paths.dev corpus/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/dev/${vars.treebank_udan_data}-dev.spacy"
      - "corpus/test/${vars.treebank_pers_data}-test.spacy"
      - "corpus/test/${vars.treebank_proi_data}-test.spacy"
      - "corpus/test/${vars.treebank_ittb_data}-test.spacy"
      - "corpus/test/${vars.treebank_llct_data}-test.spacy"
      - "corpus/test/${vars.treebank_udan_data}-test.spacy"    
      - "configs/config_trf.cfg"
    outputs:
      - "training/trf/model-best"

  - name: train-lemma_trf
    help: "Retrain tagger/parser/etc. on Latin UD treebanks and LASLA"
    script:
      - "mkdir -p training/trf/lemma"
      - >-
        python -m spacy train 
        configs/lemma_config_trf.cfg
        --output training/trf/lemma
        --gpu-id ${vars.gpu} 
        --paths.train corpus/lemma/train
        --paths.dev corpus/lemma/dev
        --nlp.lang=${vars.lang}
        --code scripts/functions.py
    deps:
      - "corpus/lemma/train/${vars.treebank_pers_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_proi_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_ittb_data}-train.spacy"      
      - "corpus/lemma/train/${vars.treebank_llct_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_udan_data}-train.spacy"
      - "corpus/lemma/train/${vars.treebank_lasl_data}-train.spacy"
      - "corpus/lemma/dev/${vars.treebank_proi_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_ittb_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_llct_data}-dev.spacy"
      - "corpus/lemma/dev/${vars.treebank_udan_data}-dev.spacy"
      - "configs/lemma_config_trf.cfg"
    outputs:
      - "training/trf/lemma/model-best"


  - name: evaluate_sm
    help: "Evaluate model on the test data and save the metrics"
    script:
      - >-
        python -m spacy evaluate 
        ./training/sm/lemma/model-best 
        ./corpus/test/
        --output ./metrics/${vars.lang}_${vars.package_name}sm_${vars.package_version}.json 
        --code scripts/functions.py
        --gpu-id ${vars.gpu}
    deps:
      - "training/sm/lemma/model-best"
      - "corpus/test/"
    outputs:
      - "metrics/${vars.lang}_${vars.package_name}sm_${vars.package_version}.json"

  - name: evaluate_md
    help: "Evaluate model on the test data and save the metrics"
    script:
      - >-
        python -m spacy evaluate 
        ./training/md/lemma/model-best 
        ./corpus/test/
        --output ./metrics/${vars.lang}_${vars.package_name}md_${vars.package_version}.json 
        --code scripts/functions.py
        --gpu-id ${vars.gpu}
    deps:
      - "training/md/lemma/model-best"
      - "corpus/test/"
    outputs:
      - "metrics/${vars.lang}_${vars.package_name}md_${vars.package_version}.json"
  - name: evaluate_lg
    help: "Evaluate model on the test data and save the metrics"
    script:
      - >-
        python -m spacy evaluate 
        ./training/lg/lemma/model-best 
        ./corpus/test/
        --output ./metrics/${vars.lang}_${vars.package_name}lg_${vars.package_version}.json 
        --code scripts/functions.py
        --gpu-id ${vars.gpu}
    deps:
      - "training/lg/lemma/model-best"
      - "corpus/test/"
    outputs:
      - "metrics/${vars.lang}_${vars.package_name}lg_${vars.package_version}.json"

  - name: evaluate_trf
    help: "Evaluate model on the test data and save the metrics"
    script:
      - >-
        python -m spacy evaluate 
        ./training/trf/lemma/model-best 
        ./corpus/test/
        --output ./metrics/${vars.lang}_${vars.package_name}trf_${vars.package_version}.json 
        --code scripts/functions.py
        --gpu-id ${vars.gpu}
    deps:
      - "training/trf/lemma/model-best"
      - "corpus/test/"
    outputs:
      - "metrics/${vars.lang}_${vars.package_name}trf_${vars.package_version}.json"

  - name: convert-ner
    help: "Convert the NER data to spaCy's binary format"
    script:
      - "mkdir -p corpus/ner/train"
      - "mkdir -p corpus/ner/dev"
      - "python scripts/convert.py ${vars.lang} assets/${vars.ud_ner_train} corpus/ner/train/ud_ner_train.spacy"
      - "python scripts/convert.py ${vars.lang} assets/${vars.ud_ner_dev} corpus/ner/dev/ud_ner_dev.spacy"
      - "python scripts/convert.py ${vars.lang} assets/${vars.herodotos_ner_train} corpus/ner/train/herodotos_ner_train.spacy"
      - "python scripts/convert.py ${vars.lang} assets/${vars.herodotos_ner_dev} corpus/ner/dev/herodotos_ner_dev.spacy"
    deps:
      - "assets/${vars.ud_ner_train}"
      - "assets/${vars.ud_ner_dev}"
      - "assets/${vars.herodotos_ner_train}"
      - "assets/${vars.herodotos_ner_dev}"
      - "scripts/convert.py"
    outputs:
      - "corpus/ner/train/ud_ner_train.spacy"
      - "corpus/ner/train/ud_ner_dev.spacy"
      - "corpus/ner/train/herodotos_ner_train.spacy"
      - "corpus/ner/train/herodotos_ner_dev.spacy"

  - name: train-ner_sm
    help: "Train the NER model for the model"
    script:
      - "mkdir -p training/sm/ner"
      - "python -m spacy train configs/ner_config_sm.cfg --output training/sm/ner --paths.train corpus/ner/train --paths.dev corpus/ner/dev --training.eval_frequency 10 --training.patience 100 --gpu-id ${vars.gpu} --code scripts/functions.py --initialize.vectors training/sm/model-best"
    deps:
      - "configs/ner_config_sm.cfg"
      - "corpus/ner/train"
      - "corpus/ner/dev"
    outputs:
      - "training/sm/ner/model-best"

  - name: train-ner_md
    help: "Train the NER model for the model"
    script:
      - "mkdir -p training/md/ner"
      - "python -m spacy train configs/ner_config_md.cfg --output training/md/ner --paths.train corpus/ner/train --paths.dev corpus/ner/dev --training.eval_frequency 10 --training.patience 100 --gpu-id ${vars.gpu} --code scripts/functions.py --initialize.vectors training/md/model-best"
    deps:
      - "configs/ner_config_md.cfg"
      - "corpus/ner/train"
      - "corpus/ner/dev"
    outputs:
      - "training/md/ner/model-best"

  - name: train-ner_lg
    help: "Train the NER model for the model"
    script:
      - "mkdir -p training/lg/ner"
      - "python -m spacy train configs/ner_config_lg.cfg --output training/lg/ner --paths.train corpus/ner/train --paths.dev corpus/ner/dev --training.eval_frequency 10 --training.patience 100 --gpu-id ${vars.gpu} --code scripts/functions.py --initialize.vectors training/lg/model-best"
    deps:
      - "configs/ner_config_lg.cfg"
      - "corpus/ner/train"
      - "corpus/ner/dev"
    outputs:
      - "training/lg/ner/model-best"     

  - name: train-ner_trf
    help: "Train the NER model for the model"
    script:
      - "mkdir -p training/trf/ner"
      - "python -m spacy train configs/ner_config_trf.cfg --output training/trf/ner --paths.train corpus/ner/train --paths.dev corpus/ner/dev --training.eval_frequency 10 --training.patience 100 --gpu-id ${vars.gpu} --code scripts/functions.py"
    deps:
      - "configs/ner_config_trf.cfg"
      - "corpus/ner/train"
      - "corpus/ner/dev"
    outputs:
      - "training/trf/ner/model-best"

  - name: assemble_sm
    help: "Assemble core model, i.e. add NER component to dep model"
    script:
      - >-
        python -m spacy assemble
        configs/config-core_sm.cfg
        training/sm/model-assembled
        --code scripts/functions.py
    deps:
      - "training/sm/model-best"
    outputs:
      - "metrics/{vars.package_name}sm-{vars.package_version}.json"
      - training/sm/model-assembled

  - name: assemble_md
    help: "Assemble core model, i.e. add NER component to dep model"
    script:
      - >-
        python -m spacy assemble
        configs/config-core_md.cfg
        training/md/model-assembled
        --code scripts/functions.py
    deps:
      - "training/md/model-best"
    outputs:
      - "metrics/{vars.package_name}md-{vars.package_version}.json"
      - training/md/model-assembled

  - name: assemble_lg
    help: "Assemble core model, i.e. add NER component to dep model"
    script:
      - >-
        python -m spacy assemble
        configs/config-core_lg.cfg
        training/lg/model-assembled
        --code scripts/functions.py
    deps:
      - "training/lg/model-best"
    outputs:
      - "metrics/{vars.package_name}lg-{vars.package_version}.json"
      - training/lg/model-assembled            

  - name: assemble_trf
    help: "Assemble core model, i.e. add NER component to dep model"
    script:
      - >-
        python -m spacy assemble
        configs/config-core_trf.cfg
        training/trf/model-assembled
        --code scripts/functions.py
    deps:
      - "training/trf/model-best"
    outputs:
      - "metrics/{vars.package_name}lg-{vars.package_version}.json"
      - training/trf/model-assembled            


  - name: assemble-meta_sm
    help: "Assemble meta.json files so that all metrics are included"
    script:
      - "python scripts/assemble_meta.py --dep_meta training/sm/model-best/meta.json --ner_meta training/sm/ner/model-best/meta.json --assembled_meta training/sm/model-assembled/meta.json"

  - name: assemble-meta_md
    help: "Assemble meta.json files so that all metrics are included"
    script:
      - "python scripts/assemble_meta.py --dep_meta training/md/model-best/meta.json --ner_meta training/md/ner/model-best/meta.json --assembled_meta training/md/model-assembled/meta.json"

  - name: assemble-meta_lg
    help: "Assemble meta.json files so that all metrics are included"
    script:
      - "python scripts/assemble_meta.py --dep_meta training/lg/model-best/meta.json --ner_meta training/lg/ner/model-best/meta.json --assembled_meta training/lg/model-assembled/meta.json"      

  - name: assemble-meta_trf
    help: "Assemble meta.json files so that all metrics are included"
    script:
      - "python scripts/assemble_meta.py --dep_meta training/trf/model-best/meta.json --ner_meta training/trf/ner/model-best/meta.json --assembled_meta training/trf/model-assembled/meta.json"      


  - name: package_sm
    help: "Package the trained core model"
    script:
      - >-
        python -m spacy package 
        training/sm/model-assembled packages 
        --name ${vars.package_name}sm 
        --version ${vars.package_version}
        --meta data/meta.json
        --code scripts/functions.py
        --force
        --build wheel
    deps:
      - "training/sm/model-assembled"
    outputs_no_cache:
      - "packages/${vars.lang}_${vars.package_name}sm-${vars.package_version}/dist/en_${vars.package_name}sm-${vars.package_version}.tar.gz"

  - name: package_md
    help: "Package the trained core model"
    script:
      - >-
        python -m spacy package 
        training/md/model-assembled packages 
        --name ${vars.package_name}md 
        --version ${vars.package_version}
        --meta data/meta.json
        --code scripts/functions.py
        --force
        --build wheel
    deps:
      - "training/md/model-assembled"
    outputs_no_cache:
      - "packages/${vars.lang}_${vars.package_name}md-${vars.package_version}/dist/en_${vars.package_name}md-${vars.package_version}.tar.gz"      

  - name: package_lg
    help: "Package the trained core model"
    script:
      - >-
        python -m spacy package 
        training/lg/model-assembled packages 
        --name ${vars.package_name}lg 
        --version ${vars.package_version}
        --meta data/meta.json
        --code scripts/functions.py
        --force
        --build wheel
    deps:
      - "training/lg/model-assembled"
    outputs_no_cache:
      - "packages/${vars.lang}_${vars.package_name}lg-${vars.package_version}/dist/en_${vars.package_name}lg-${vars.package_version}.tar.gz"                    

  - name: package_trf
    help: "Package the trained core model"
    script:
      - >-
        python -m spacy package 
        training/trf/model-assembled packages 
        --name ${vars.package_name}trf 
        --version ${vars.package_version}
        --meta data/meta.json
        --code scripts/functions.py
        --force
        --build wheel
    deps:
      - "training/trf/model-assembled"
    outputs_no_cache:
      - "packages/${vars.lang}_${vars.package_name}trf-${vars.package_version}/dist/en_${vars.package_name}trf-${vars.package_version}.tar.gz"                    

  - name: document
    help: "Document ${vars.package_name}xx"
    script:
      - >-
        python -m spacy project document 
        --output README.md
    outputs:
      - "README.md"

  - name: clean
    help: "Remove intermediate files"
    script:
      - sh -c "rm -rf assets/original/*"      
      - sh -c "rm -rf assets/processed/*"
      - sh -c "rm -rf assets/preprocess/*"
      - sh -c "rm -rf corpus/*"
      - sh -c "rm -rf metrics/*"
      - sh -c "rm -rf training/*"
      - sh -c "rm -rf vectors/*"