E2E test orchestration with Gulp on Windows: Unable to kill process(es)

Question

What I'm trying to achieve

This question is related to another one I recently closed with a horrible hack™.

I am trying to write a script that can be used a step in a context of a CI/build pipeline.

The script is supposed to run Protractor-based end-to-end tests for our Angular single-page application (SPA).

The script is required to do the following actions (in order):

1. run a .NET Core microservice called "App"
2. run a .NET Core microservice called "Web"
3. run the SPA
4. run a command that executes Protractor tests
5. after steps 4 is complete (either successfully or with an error), terminate processes created on steps 1-3. This is absolutely necessary otherwise the build will never finish in CI and/or there will be zombie Web/App/SPA processes which will break future build pipeline execution.

The issue

I haven't started working on step 4 ("e2e test") because I really want to make sure that the step 5 ("cleanup") works as intended.

As you could guess (right), the cleanup step does not work. Specifically, the processes "App" and "Web" do not get killed for some reason and continue running.

BTW, I made sure that my gulp script is executed with elevated (admin) privileges.

Issue - UPDATE 1

I have just discovered the direct cause of the issue (I think), I don't know what's the root cause though. There are 5 processes launched instead of 1 as I was expecting. E.g., for App process the following processes are observed in Process manager:

{                          
  "id": 14840,             
  "binary": "cmd.exe",     
  "title": "Console"       
},                         
{                          
  "id": 12600,             
  "binary": "dotnet.exe",  
  "title": "Console"       
},                         
{                          
  "id": 12976,             
  "binary": "cmd.exe",     
  "title": "Console"       
},                         
{                          
  "id": 5492,              
  "binary": "cmd.exe",     
  "title": "Console"       
},                         
{                          
  "id": 2636,              
  "binary": "App.exe",
  "title": "Console"       
}                          

Similarly, five processes rather than one are created for Web service:

{                          
  "id": 13264,             
  "binary": "cmd.exe",     
  "title": "Console"       
},                         
{                          
  "id": 1900,              
  "binary": "dotnet.exe",  
  "title": "Console"       
},                         
{                          
  "id": 4668,              
  "binary": "cmd.exe",     
  "title": "Console"       
},                         
{                          
  "id": 15520,             
  "binary": "Web.exe",
  "title": "Console"       
},                         
{                          
  "id": 7516,              
  "binary": "cmd.exe",     
  "title": "Console"       
}                          

How I am doing that

Basically, the work horse here is the runCmdAndListen() function that spins off the processes by running the cmd provided as an argument. When the function launches a process be the means of Node.js's exec(), it is then pushed to the createdProcesses array for tracking.

The Gulp step called CLEANUP = "cleanup" is responsible for iterating through the createdProcesses and invoking .kill('SIGTERM') on each of them, which is supposed to kill all the processes created earlier.

gulpfile.js (Gulp task script)

Imports and constants

const gulp = require('gulp');
const exec = require('child_process').exec;
const path = require('path');

const RUN_APP = `run-app`;
const RUN_WEB = `run-web`;
const RUN_SPA = `run-spa`;
const CLEANUP = `cleanup`;

const appDirectory = path.join(`..`, `App`);
const webDirectory = path.join(`..`, `Web`);
const spaDirectory = path.join(`.`);

const createdProcesses = [];

runCmdAndListen()

/**
 * Runs a command and taps on `stdout` waiting for a `resolvePhrase` if provided.
 * @param {*} name Title of the process to use in console output.
 * @param {*} command Command to execute.
 * @param {*} cwd Command working directory.
 * @param {*} env Command environment parameters.
 * @param {*} resolvePhrase Phrase to wait for in `stdout` and resolve on.
 * @param {*} rejectOnError Flag showing whether to reject on a message in `stderr` or not.
 */
function runCmdAndListen(name, command, cwd, env, resolvePhrase, rejectOnError) {

  const options = { cwd };
  if (env) options.env = env;

  return new Promise((resolve, reject) => {
    const newProcess = exec(command, options);

    console.info(`Adding a running process with id ${newProcess.pid}`);
    createdProcesses.push({ childProcess: newProcess, isRunning: true });

    newProcess.on('exit', () => {
      createdProcesses
        .find(({ childProcess, _ }) => childProcess.pid === newProcess.pid)
        .isRunning = false;
    });

    newProcess.stdout
      .on(`data`, chunk => {
        if (resolvePhrase && chunk.toString().indexOf(resolvePhrase) >= 0) {
          console.info(`RESOLVED ${name}/${resolvePhrase}`);
          resolve();
        }
      });

    newProcess.stderr
      .on(`data`, chunk => {
        if (rejectOnError) reject(chunk);
      });

    if (!resolvePhrase) {
      console.info(`RESOLVED ${name}`);
      resolve();
    }
  });
}

Basic Gulp tasks

gulp.task(RUN_APP, () => runCmdAndListen(
  `[App]`,
  `dotnet run --no-build --no-dependencies`,
  appDirectory,
  { 'ASPNETCORE_ENVIRONMENT': `Development` },
  `Now listening on:`,
  true)
);

gulp.task(RUN_WEB, () => runCmdAndListen(
  `[Web]`,
  `dotnet run --no-build --no-dependencies`,
  webDirectory,
  { 'ASPNETCORE_ENVIRONMENT': `Development` },
  `Now listening on:`,
  true)
);

gulp.task(RUN_SPA, () => runCmdAndListen(
  `[SPA]`,
  `npm run start-prodish-for-e2e`,
  spaDirectory,
  null,
  `webpack: Compiled successfully
  `,
  false)
);

gulp.task(CLEANUP, () => {
  createdProcesses
    .forEach(({ childProcess, isRunning }) => {
      console.warn(`Killing child process ${childProcess.pid}`);

      // if (isRunning) {
      childProcess.kill('SIGTERM');
      // }
    });
});

The orchestrating task

gulp.task(
  'e2e',
  gulp.series(
    gulp.series(
      RUN_APP,
      RUN_WEB,
    ),
    RUN_SPA,
    CLEANUP,
  ),
  () => console.info(`All tasks complete`),
);

gulp.task('default', gulp.series('e2e'));

Answer 1

• dotnet run does not propagate the kill signal to children in Windows (that's the behavior in Windows, which differs in POSIX OS), you're doing the right thing, which is to manage the children processes
• however SIGTERM won't work on windows: nodejs doc process_signal_events; that's your problem. You might want to try SIGKILL.
• for a pure js solution, just process.kill(process.pid, SIGKILL) though, this might need to be tested: nodejs issue 12378.

• for a reliable MSFT tested solution (but not cross-platform), consider powershell to manage trees thanks to the following powershell functions:

  function startproc($mydotnetcommand)
  {
      $parentprocid = Start-Process $mydotnetcommand -passthru
  }
  
  function stopproctree($parentprocid)
  {
      $childpidlist= Get-WmiObject win32_process |`
          where {$_.ParentProcessId -eq $parentprocid}
      Get-Process -Id $childpidlist -ErrorAction SilentlyContinue |`
          Stop-Process -Force
  }
  

  You can use the 2nd function from the outside of the ps script through passing parent PID as argument to the stopproctree function with:

  param([Int32]$parentprocid)
  stopproctree $parentprocid
  

  (in the script, say treecleaner.ps1), and then powershell.exe -file treecleaner.ps1 -parentprocid XXX