run.sh

# Copyright 2022 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#!/bin/bash

# Create a virtual environment
conda deactivate
conda update conda -y
conda update anaconda -y
pip install --upgrade pip
python3 -m pip install --user virtualenv
conda create -n strata python=3.9 -y
conda activate strata
# Install all necessary packages
pip install transformers
pip install -r requirements.txt

# Download and prepare data
WORK_DIR="/tmp/strata"
rm -rf "${WORK_DIR}" && mkdir -p "${WORK_DIR}"
wget https://storage.googleapis.com/gresearch/strata/demo.zip -P "${WORK_DIR}"
DEMO_ZIP_FILE="${WORK_DIR}/demo.zip"
unzip "${DEMO_ZIP_FILE}" -d "${WORK_DIR}" && rm "${DEMO_ZIP_FILE}"
MODEL_DIR="${WORK_DIR}/demo/bert-base-synnli-from-scitail"
DATA_DIR="${WORK_DIR}/demo/scitail-8"
OUTPUT_DIR="/tmp/output"
rm -rf "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}"

# Specific hyperparameters
DEBUG_MODE_ON=True  # Turn this off for the actual demo
if [[ "${DEBUG_MODE_ON}" == "True" ]];
then
  LAUNCH_SCRIPT="python -c"
  MAX_SELFTRAIN_ITERATIONS=1
  TRAIN_FILE="train.csv"
  INFER_FILE="eval_16.csv"
  EVAL_FILE="eval_16.csv"
  MAX_STEPS=2
else
  NUM_NODES=1
  NUM_TRAINERS=4
  LAUNCH_SCRIPT="torchrun --nnodes='${NUM_NODES}' --nproc_per_node='${NUM_TRAINERS}' python -c"
  MAX_SELFTRAIN_ITERATIONS=100
  TRAIN_FILE="train.csv"
  INFER_FILE="infer.csv"
  EVAL_FILE="eval_256.csv"
  MAX_STEPS=100000
fi

# Start self-training
${LAUNCH_SCRIPT} "
import os
from selftraining import selftrain

data_dir = '${DATA_DIR}'
parameters_dict = {
  'max_selftrain_iterations': ${MAX_SELFTRAIN_ITERATIONS},
  'model_name_or_path': '${MODEL_DIR}',
  'output_dir': '${OUTPUT_DIR}',
  'train_file': os.path.join(data_dir, '${TRAIN_FILE}'),
  'infer_file': os.path.join(data_dir, '${INFER_FILE}'),
  'eval_file': os.path.join(data_dir, '${EVAL_FILE}'),
  'evaluation_strategy': 'steps',
  'task_name': 'scitail',
  'label_list': ['entails', 'neutral'],
  'per_device_train_batch_size': 32,
  'per_device_eval_batch_size': 8,
  'max_length': 128,
  'learning_rate': 2e-5,
  'max_steps': ${MAX_STEPS},
  'eval_steps': 1,
  'early_stopping_patience': 50,
  'overwrite_output_dir': True,
  'do_filter_by_confidence': False,
  'do_filter_by_val_performance': True,
  'finetune_on_labeled_data': False,
  'seed': 42,
}

selftrain(**parameters_dict)
"