Dockerfile_CSE468

.gitignore

@@ -10,6 +10,9 @@ courselabs/*
 # config
 config.py
 
+local
+courselabs/*
+
 # Virtualenv
 .Python
 bin

Dockerfile

@@ -1,5 +1,5 @@
 # Start with empty ubuntu machine
-FROM ubuntu:15.04
+FROM ubuntu:18.04
 
 MAINTAINER Autolab Development Team "autolab-dev@andrew.cmu.edu"
 
@@ -15,9 +15,8 @@ WORKDIR /opt/TangoService/Tango
 RUN mkdir volumes
 
 WORKDIR /opt
-
 # Install dependancies
-RUN apt-get update && apt-get install -y \
+RUN apt-get update --fix-missing && DEBIAN_FRONTEND=nointeractive apt-get install -y \
 	nginx \
 	curl \
 	git \
@@ -68,7 +67,6 @@ RUN cp /opt/TangoService/Tango/deployment/config/redis.conf /etc/redis.conf
 # Reload new config scripts
 CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
 
-
 # TODO: 
 # volumes dir in root dir, supervisor only starts after calling start once , nginx also needs to be started
 # Different log numbers for two different tangos

clients/README.md

@@ -1,7 +1,7 @@
 This directory contains the Tango client program and some example jobs
 that are useful for testing.
 
-tango-cli.py - Tango client program
+tango-rest.py - Tango client program
 
 Example jobs:
 job1 	  - simple hello job

clients/tango-cli.py → clients/tango-rest.py

@@ -1,7 +1,7 @@
 #!/usr/local/bin/python
 #
 #
-# tango-cli.py - Command line client for the RESTful Tango.
+# tango-rest.py - Command line client for the RESTful Tango.
 #
 
 import os

config.template.py

@@ -77,7 +77,9 @@ class Config:
     # Default value of '*'' points this path to /path/to/Tango/volumes/
     DOCKER_VOLUME_PATH = '*'
     DOCKER_HOST_USER = ''
-
+    DOCKER_DISTRIBUTED_HOSTS = ['host.example.domain','host2.example.domain','host3.example.domain']
+    USE_SSH_MASTER = True
+
     # Maximum size for input files in bytes
     MAX_INPUT_FILE_SIZE = 250 * 1024 * 1024 # 250MB
 
@@ -86,6 +88,8 @@ class Config:
 
     # VM ulimit values
     VM_ULIMIT_FILE_SIZE = 100 * 1024 * 1024
+    #(Ram specified in MB)
+    VM_ULIMIT_USER_RAM = 512
     VM_ULIMIT_USER_PROC = 100
 
     # How many times to reschedule a failed job
@@ -127,7 +131,7 @@ class Config:
     ######
     # Part 4: Settings for shared memory
     #
-    USE_REDIS = True
+    USE_REDIS = False
     REDIS_HOSTNAME = "127.0.0.1"
     REDIS_PORT = 6379
 

jobManager.py

@@ -65,44 +65,44 @@ def __manage(self):
                 if not job.accessKey and Config.REUSE_VMS:
                     id, vm = self.jobQueue.getNextPendingJobReuse(id)
                     job = self.jobQueue.get(id)
-
-                try:
-                    # Mark the job assigned
-                    self.jobQueue.assignJob(job.id)
-                    # if the job has specified an account
-                    # create an VM on the account and run on that instance
-                    if job.accessKeyId:
-                        from vmms.ec2SSH import Ec2SSH
-                        vmms = Ec2SSH(job.accessKeyId, job.accessKey)
-                        newVM = copy.deepcopy(job.vm)
-                        newVM.id = self._getNextID()
-                        preVM = vmms.initializeVM(newVM)
-                    else:
-                        # Try to find a vm on the free list and allocate it to
-                        # the worker if successful.
-                        if Config.REUSE_VMS:
-                            preVM = vm
+                if job:
+                    try:
+                        # Mark the job assigned
+                        self.jobQueue.assignJob(job.id)
+                        # if the job has specified an account
+                        # create an VM on the account and run on that instance
+                        if job.accessKeyId:
+                            from vmms.ec2SSH import Ec2SSH
+                            vmms = Ec2SSH(job.accessKeyId, job.accessKey)
+                            newVM = copy.deepcopy(job.vm)
+                            newVM.id = self._getNextID()
+                            preVM = vmms.initializeVM(newVM)
                         else:
-                            preVM = self.preallocator.allocVM(job.vm.name)
-                        vmms = self.vmms[job.vm.vmms]  # Create new vmms object
-
-                    # Now dispatch the job to a worker
-                    self.log.info("Dispatched job %s:%d to %s [try %d]" %
-                                  (job.name, job.id, preVM.name, job.retries))
-                    job.appendTrace(
-                        "%s|Dispatched job %s:%d [try %d]" %
-                        (datetime.utcnow().ctime(), job.name, job.id, job.retries))
-
-                    Worker(
-                        job,
-                        vmms,
-                        self.jobQueue,
-                        self.preallocator,
-                        preVM
-                    ).start()
-
-                except Exception as err:
-                    self.jobQueue.makeDead(job.id, str(err))
+                            # Try to find a vm on the free list and allocate it to
+                            # the worker if successful.
+                            if Config.REUSE_VMS:
+                                preVM = vm
+                            else:
+                                preVM = self.preallocator.allocVM(job.vm.name)
+                            vmms = self.vmms[job.vm.vmms]  # Create new vmms object
+
+                        # Now dispatch the job to a worker
+                        self.log.info("Dispatched job %s:%d to %s [try %d]" %
+                                      (job.name, job.id, preVM.name, job.retries))
+                        job.appendTrace(
+                            "%s|Dispatched job %s:%d [try %d]" %
+                            (datetime.utcnow().ctime(), job.name, job.id, job.retries))
+
+                        Worker(
+                            job,
+                            vmms,
+                            self.jobQueue,
+                            self.preallocator,
+                            preVM
+                        ).start()
+
+                    except Exception as err:
+                        self.jobQueue.makeDead(job.id, str(err))
 
             # Sleep for a bit and then check again
             time.sleep(Config.DISPATCH_PERIOD)

vmms/Dockerfile

@@ -1,12 +1,59 @@
 # Autolab - autograding docker image
 
-FROM ubuntu:14.04
-MAINTAINER Mihir Pandya <mihir.m.pandya@gmail.com>
+FROM ubuntu:18.04
+MAINTAINER David Dobmeier <daviddob@buffalo.edu>
 
-RUN apt-get update --fix-missing
+#C++ Setup
+RUN apt-get update
 RUN apt-get install -y gcc
 RUN apt-get install -y make
 RUN apt-get install -y build-essential
+RUN apt-get install -y libcunit1-dev libcunit1-doc libcunit1
+
+#Python Setup
+RUN apt-get update --fix-missing && \
+    DEBIAN_FRONTEND=nointeractive apt-get install -y \
+    python3 python3-numpy python3-nose python3-pandas \
+    python python-numpy python-nose python-pandas \
+    pep8 python-pip python3-pip python-wheel \
+    python-sphinx && \
+    pip install --upgrade setuptools
+
+#Java Setup
+RUN apt-get update --fix-missing
+RUN apt-get install -y default-jdk
+
+#Valgrind Setup
+RUN apt-get update
+RUN apt-get install -y valgrind
+
+#SML Setup
+RUN mkdir -p /usr/local/bin/sml
+WORKDIR /usr/local/bin/sml
+RUN apt-get install -y gcc-multilib g++-multilib lib32z1 lib32ncurses5 wget
+RUN wget http://www.smlnj.org/dist/working/110.78/config.tgz
+RUN tar -xzvf config.tgz
+RUN config/install.sh
+RUN ln -s /usr/local/bin/sml/bin/sml /usr/local/sbin/sml
+RUN ln -s /usr/local/bin/sml/bin/ml-lex /usr/local/sbin/ml-lex
+RUN ln -s /usr/local/bin/sml/bin/ml-yacc /usr/local/sbin/ml-yacc
+
+#Flex setup
+RUN apt-get install -y flex
+
+#Bison Setup
+RUN apt-get install -y bison
+
+#Utility setup
+RUN apt-get install -y unzip
+
+#NodeJS setup
+RUN apt-get update --fix-missing
+RUN apt-get install -y nodejs
+RUN apt-get install -y npm
+
+#OCaml Setup
+RUN apt-get install -y ocaml
 
 # Install autodriver
 WORKDIR /home
@@ -16,6 +63,7 @@ RUN mkdir autolab autograde output
 RUN chown autolab:autolab autolab
 RUN chown autolab:autolab output
 RUN chown autograde:autograde autograde
+RUN apt-get update && apt-get install -y sudo 
 RUN apt-get install -y git
 RUN git clone https://github.com/autolab/Tango.git
 WORKDIR Tango/autodriver
@@ -25,10 +73,12 @@ RUN chmod +s /usr/bin/autodriver
 
 # Clean up
 WORKDIR /home
-RUN apt-get remove -y git
 RUN apt-get -y autoremove
 RUN rm -rf Tango/
 
 # Check installation
 RUN ls -l /home
 RUN which autodriver
+RUN which javac
+RUN which sml
+RUN g++ --version

vmms/Dockerfile_CSE191

@@ -0,0 +1,62 @@
+# Autolab - autograding docker image
+
+FROM ubuntu:18.04
+MAINTAINER Matthew Knepley <knepley@buffalo.edu>
+
+#C Setup
+RUN apt-get update
+RUN apt-get install -y gcc
+RUN apt-get install -y make
+RUN apt-get install -y build-essential
+RUN apt-get install -y libcunit1-dev libcunit1-doc libcunit1
+
+#Python Setup
+RUN apt-get update --fix-missing && \
+    DEBIAN_FRONTEND=nointeractive apt-get install -y \
+    python3 python3-numpy python3-nose python3-pandas \
+    python python-numpy python-nose python-pandas \
+    pep8 python-pip python3-pip python-wheel \
+    python-sphinx && \
+    pip install --upgrade setuptools
+
+#Valgrind Setup
+RUN apt-get update
+RUN apt-get install -y valgrind
+#OPAM Setup
+RUN apt-get install -y m4
+RUN apt-get install -y pkg-config
+RUN apt-get install -y opam
+
+#Utility setup
+RUN apt-get install -y unzip
+
+# Install autodriver
+WORKDIR /home
+RUN useradd autolab
+RUN useradd autograde
+RUN mkdir autolab autograde output
+RUN chown autolab:autolab autolab
+RUN chown autolab:autolab output
+RUN chown autograde:autograde autograde
+RUN apt-get update && apt-get install -y sudo
+RUN apt-get install -y git
+RUN git clone https://github.com/autolab/Tango.git
+WORKDIR Tango/autodriver
+RUN make clean && make
+RUN cp autodriver /usr/bin/autodriver
+RUN chmod +s /usr/bin/autodriver
+
+# Install Coq
+RUN su autograde -c "opam init && opam install coq"
+RUN echo "PATH=${PATH}:/home/autograde/.opam/system/bin" >> /home/autograde/.profile
+
+# Clean up
+WORKDIR /home
+RUN apt-get -y autoremove
+RUN rm -rf Tango/
+
+# Check installation
+RUN ls -l /home
+RUN which autodriver
+RUN su autograde -c "/bin/bash -c 'source ~/.profile && which coqc'"
+RUN gcc --version

vmms/Dockerfile_CSE421

@@ -0,0 +1,64 @@
+FROM ubuntu:16.04
+MAINTAINER Farshad Ghanei <farshadg@buffalo.edu>
+#prerequisites
+RUN apt-get update --fix-missing
+RUN apt-get update && apt-get install -y apt-utils
+RUN apt-get install -y gcc make build-essential libcunit1-dev libcunit1-doc libcunit1 wget python qemu xorg-dev libncurses5-dev gdb git
+# Install autodriver
+WORKDIR /home
+RUN useradd autolab
+RUN useradd autograde
+RUN mkdir autolab autograde output
+RUN chown autolab:autolab autolab
+RUN chown autolab:autolab output
+RUN chown autograde:autograde autograde
+RUN apt-get update && apt-get install -y sudo
+RUN apt-get install -y git
+RUN git clone https://github.com/autolab/Tango.git
+WORKDIR Tango/autodriver
+RUN make clean && make
+RUN cp autodriver /usr/bin/autodriver
+RUN chmod +s /usr/bin/autodriver
+###############################################
+# configuraion and setup for bochs and pintos #
+###############################################
+ENV PINTOSDIR /home/autograde/pintos_base
+ENV DSTDIR /usr/local
+ENV SRCDIR $PINTOSDIR/sources
+RUN mkdir -p $SRCDIR
+RUN mkdir -p $PINTOSDIR
+RUN mkdir -p $DSTDIR/bin
+ENV BXSHARE $DSTDIR/share/bochs
+ENV PATH="${DSTDIR}/bin:${PATH}"
+
+# These files may be put on a local server, but keep the folder structure
+WORKDIR $SRCDIR/
+RUN git clone git://pintos-os.org/pintos-anon
+RUN mv pintos-anon/* $PINTOSDIR
+#RUN wget http://web.stanford.edu/class/cs140/projects/pintos/pintos.tar.gz
+#RUN tar -xzf pintos.tar.gz
+#RUN mv pintos/* $PINTOSDIR
+
+WORKDIR $SRCDIR/
+RUN wget http://www.oldlinux.org/Linux.old/bochs/Bochs/bochs-2.2.6/bochs-2.2.6.tar.gz
+#RUN wget http://web.stanford.edu/class/cs140/projects/pintos/bochs-2.2.6.tar.gz
+
+WORKDIR $PINTOSDIR/src/misc/
+RUN ./bochs-2.2.6-build.sh
+
+WORKDIR $PINTOSDIR/src/utils/
+RUN sed -i "5i GDBMACROS=$PINTOSDIR/src/misc/gdb-macros" pintos-gdb
+RUN make
+RUN cp backtrace pintos pintos-gdb pintos-mkdisk Pintos.pm pintos-set-cmdline squish-pty squish-unix $DSTDIR/bin
+
+RUN chown -R autograde:autograde /home/autograde
+RUN chown -R autograde:autograde $BXSHARE
+RUN chown -R autograde:autograde $DSTDIR/bin
+# Clean up
+WORKDIR /home
+RUN apt-get -y autoremove
+RUN rm -rf Tango/
+# Check installation
+RUN ls -l /home
+RUN which autodriver
+