c# online compiling and execution security issues

Question

So I was thinking of writing online c# compiler and execution environment. And of course problem #1 is security. I ended up creating a little-privileged appdomain for user code and starting it in a new process which is tightly monitored for cpu and memory consumption. Standard console application namespaces are available. So my question is this: can you think of ways of breaking something in some way? You can try your ideas on the spot rundotnet.

Edit2 If anyone cares about the code, there is now open source fork of this project: rextester at github

Edit1 As a response to one of the comments here are some code samples.

So basically you create a console application. I'll just post a big chunk of it:

class Sandboxer : MarshalByRefObject
{
    private static object[] parameters = { new string[] { "parameter for the curious" } };

    static void Main(string[] args)
    {
        Console.OutputEncoding = Encoding.UTF8;
        string pathToUntrusted = args[0].Replace("|_|", " ");
        string untrustedAssembly = args[1];
        string entryPointString = args[2];
        string[] parts = entryPointString.Split(new string[] { "|" }, StringSplitOptions.RemoveEmptyEntries);
        string name_space = parts[0];
        string class_name =  parts[1];
        string method_name = parts[2];

        //Setting the AppDomainSetup. It is very important to set the ApplicationBase to a folder 
        //other than the one in which the sandboxer resides.
        AppDomainSetup adSetup = new AppDomainSetup();
        adSetup.ApplicationBase = Path.GetFullPath(pathToUntrusted);

        //Setting the permissions for the AppDomain. We give the permission to execute and to 
        //read/discover the location where the untrusted code is loaded.
        PermissionSet permSet = new PermissionSet(PermissionState.None);
        permSet.AddPermission(new SecurityPermission(SecurityPermissionFlag.Execution));


        //Now we have everything we need to create the AppDomain, so let's create it.
        AppDomain newDomain = AppDomain.CreateDomain("Sandbox", null, adSetup, permSet, null);


        //Use CreateInstanceFrom to load an instance of the Sandboxer class into the
        //new AppDomain. 
        ObjectHandle handle = Activator.CreateInstanceFrom(
            newDomain, typeof(Sandboxer).Assembly.ManifestModule.FullyQualifiedName,
            typeof(Sandboxer).FullName
            );
        //Unwrap the new domain instance into a reference in this domain and use it to execute the 
        //untrusted code.
        Sandboxer newDomainInstance = (Sandboxer)handle.Unwrap();

        Job job = new Job(newDomainInstance, untrustedAssembly, name_space, class_name, method_name, parameters);
        Thread thread = new Thread(new ThreadStart(job.DoJob));
        thread.Start();
        thread.Join(10000);
        if (thread.ThreadState != ThreadState.Stopped)
        {
            thread.Abort();
            Console.Error.WriteLine("Job taking too long. Aborted.");
        }
        AppDomain.Unload(newDomain);
    }

    public void ExecuteUntrustedCode(string assemblyName, string name_space, string class_name, string method_name, object[] parameters)
    {
        MethodInfo target = null;
        try
        {
            target = Assembly.Load(assemblyName).GetType(name_space+"."+class_name).GetMethod(method_name);
            if (target == null)
                throw new Exception();
        }
        catch (Exception)
        {
            Console.Error.WriteLine("Entry method '{0}' in class '{1}' in namespace '{2}' not found.", method_name, class_name, name_space);
            return;
        }

        ...            

        //Now invoke the method.
        try
        {
            target.Invoke(null, parameters);
        }
        catch (Exception e)
        {
            ...
        }
    }
}

class Job
{
    Sandboxer sandboxer = null;
    string assemblyName;
    string name_space;
    string class_name;
    string method_name;
    object[] parameters;

    public Job(Sandboxer sandboxer, string assemblyName, string name_space, string class_name, string method_name, object[] parameters)
    {
        this.sandboxer = sandboxer;
        this.assemblyName = assemblyName;
        this.name_space = name_space;
        this.class_name = class_name;
        this.method_name = method_name;
        this.parameters = parameters;
    }

    public void DoJob()
    {
        try
        {
            sandboxer.ExecuteUntrustedCode(assemblyName, name_space, class_name, method_name, parameters);
        }
        catch (Exception e)
        {
            Console.Error.WriteLine(e.Message);
        }
    }
}

You compile the above and have executable which you start and monitor in a new process:

using (Process process = new Process())
{
    try
    {
        double TotalMemoryInBytes = 0;
        double TotalThreadCount = 0;
        int samplesCount = 0;

        process.StartInfo.FileName = /*path to sandboxer*/;
        process.StartInfo.Arguments = folder.Replace(" ", "|_|") + " " + assemblyName + " Rextester|Program|Main"; //assemblyName - assembly that contains compiled user code
        process.StartInfo.UseShellExecute = false;
        process.StartInfo.CreateNoWindow = true;
        process.StartInfo.RedirectStandardOutput = true;
        process.StartInfo.RedirectStandardError = true;

        DateTime start = DateTime.Now;
        process.Start();

        OutputReader output = new OutputReader(process.StandardOutput);
        Thread outputReader = new Thread(new ThreadStart(output.ReadOutput));
        outputReader.Start();
        OutputReader error = new OutputReader(process.StandardError);
        Thread errorReader = new Thread(new ThreadStart(error.ReadOutput));
        errorReader.Start();


        do
        {
            // Refresh the current process property values.
            process.Refresh();
            if (!process.HasExited)
            {
                try
                {
                    var proc = process.TotalProcessorTime;
                    // Update the values for the overall peak memory statistics.
                    var mem1 = process.PagedMemorySize64;
                    var mem2 = process.PrivateMemorySize64;

                    //update stats
                    TotalMemoryInBytes += (mem1 + mem2);
                    TotalThreadCount += (process.Threads.Count);
                    samplesCount++;

                    if (proc.TotalSeconds > 5 || mem1 + mem2 > 100000000 || process.Threads.Count > 100 || start + TimeSpan.FromSeconds(10) < DateTime.Now)
                    {
                        var time = proc.TotalSeconds;
                        var mem = mem1 + mem2;
                        process.Kill();

                        ...
                    }
                }
                catch (InvalidOperationException)
                {
                    break;
                }
            }
        }
        while (!process.WaitForExit(10)); //check process every 10 milliseconds
        process.WaitForExit();
        ...
}

...

class OutputReader
{
    StreamReader reader;
    public string Output
    {
        get;
        set;
    }
    StringBuilder sb = new StringBuilder();
    public StringBuilder Builder
    {
        get
        {
            return sb;
        }
    }
    public OutputReader(StreamReader reader)
    {
        this.reader = reader;
    }

    public void ReadOutput()
    {
        try
        {                
            int bufferSize = 40000;
            byte[] buffer = new byte[bufferSize];
            int outputLimit = 200000;
            int count;
            bool addMore = true;
            while (true)
            {
                Thread.Sleep(10);
                count = reader.BaseStream.Read(buffer, 0, bufferSize);
                if (count != 0)
                {
                    if (addMore)
                    {
                        sb.Append(Encoding.UTF8.GetString(buffer, 0, count));
                        if (sb.Length > outputLimit)
                        {
                            sb.Append("\n\n...");
                            addMore = false;
                        }
                    }
                }
                else
                    break;
            }
            Output = sb.ToString();
        }
        catch (Exception e)
        {
           ...
        }
    }
}

Assemblies that user code can use are added at compile time:

CompilerParameters cp = new CompilerParameters();
cp.GenerateExecutable = false;
cp.OutputAssembly = ...
cp.GenerateInMemory = false;
cp.TreatWarningsAsErrors = false;
cp.WarningLevel = 4;
cp.IncludeDebugInformation = false;

cp.ReferencedAssemblies.Add("System.dll");
cp.ReferencedAssemblies.Add("System.Core.dll");
cp.ReferencedAssemblies.Add("System.Data.dll");
cp.ReferencedAssemblies.Add("System.Data.DataSetExtensions.dll");
cp.ReferencedAssemblies.Add("System.Xml.dll");
cp.ReferencedAssemblies.Add("System.Xml.Linq.dll");

using (CodeDomProvider provider = CodeDomProvider.CreateProvider(/*language*/))
{
    cr = provider.CompileAssemblyFromSource(cp, new string[] { data.Program });
}

Answer 1

Have you looked at Mono's Compiler as a service? I think that is pretty cool what they are doing, perhaps something there could be useful to you for this project.

Answer 2

For a good example of something similar to this already in existence, there is a place at http://www.topcoder.com, which has an "Algorithm Arena", where code is submitted and automatically scored. There are restrictions against using certain types of classes, such as Exception, but it may be a good idea to examine their application for a proof of concept.