Gulp: Use output from one task in another task

Question

I have have three gulp tasks, where the last task (allScripts) runs the two dependent tasks first and then joins the resultant files from them.

I could, in the last task, delete the two result-files from the two first tasks and live happily ever after with the joined file.

But I was thinking, is it possible to avoid the two temporary files by somehow piping them into the allScripts task "directly"?

gulp.task('firstGroup', function() {
  return gulp.src('some/files/*.js')
    .pipe(doSomething())
    .pipe(concat('some-scripts.js'))
    .pipe(gulp.dest('dest'));
});

gulp.task('secondGroup', function() {
  return gulp.src('some/other/files/*.js')
    .pipe(doSomethingElse())
    .pipe(concat('some-other-scripts.js'))
    .pipe(gulp.dest('dest'));
});

gulp.task('allScripts', ['firstGroup','secondGroup'], function() {
  return gulp.src(['dest/some-scripts.js','dest/some-other-scripts.js'])
    .pipe(concat('all-scripts.js'))
    .pipe(gulp.dest('dest'))
    // delete the two src-files
});

Answer 1

If everything can be a single task, you can use the gulp-merge plugin to combine multiple streams into one. There is also a solution below if the tasks need to stay separate, but please note that that method is a hack because it relies on a exposed property in Gulp.

Without a hack solution, using the output from one task in another, would require intermediary storage, like what you are doing with a file.

Single task solution:

Here is a barebones demo using gulp-merge:

var gulp = require('gulp');
var gulpMerge = require('gulp-merge');
var concat = require('gulp-concat');
var replace = require('gulp-replace');

gulp.task('all-txt', function() {
    return gulpMerge(
            gulp.src('file1.txt')
                .pipe(replace(/foo/g, 'bar')),
            gulp.src('file2.txt')
                .pipe(replace(/baz/g, 'qux'))
        )
        .pipe(concat('all-text.txt'))
        .pipe(gulp.dest('dest'));
});

---

In your case and using the code in your question it would look like:

var gulp = require('gulp');
var gulpMerge = require('gulp-merge');
var concat = require('gulp-concat');
// ... your plugins

gulp.task('allScripts', function() {
    return gulpMerge(
            gulp.src('some/files/*.js')
                .pipe(doSomething())
                .pipe(concat('some-scripts.js')),
            gulp.src('some/other/files/*.js')
                .pipe(doSomethingElse())
                .pipe(concat('some-other-scripts.js'))
        )
        .pipe(concat('all-scripts.js'))
        .pipe(gulp.dest('dest'));
});

---

Multiple task solution:

If your task structure is such that you can not merge them into a single task using the method above, this is your best bet. It is a bit hacky in the sense that it relies on Gulp.tasks which is a non-standard exposed property. There is no gurantee that this will work with future versions of Gulp (currently tested with Gulp v3.8.10).

This snippet relies on the event-stream package because it is more robust and I use some of their utilities in the runTasksAndGetStreams function.

var gulp = require('gulp');

var concat = require('gulp-concat');
var replace = require('gulp-replace');
var es = require('event-stream');


gulp.task('all-txt', function() {
    return es.merge.apply(null, runTasksAndGetStreams(['file1-txt', 'file2-txt']))
        .pipe(concat('all-text.txt'))
        .pipe(gulp.dest('dest'));
});


gulp.task('file1-txt', ['random-task-dep'], function() {
    return gulp.src('file1.txt')
        .pipe(replace(/foo/g, 'bar'));
});

gulp.task('file2-txt', function() {
    return gulp.src('file2.txt')
        .pipe(replace(/baz/g, 'qux'));
});

gulp.task('random-task-dep', function() {
    return gulp.src('random-file.txt')
        .pipe(gulp.dest('dest'));
});



// Run the given tasks and returns their streams
// This will also take care of any task dependencies
//
// This is basically a custom gulp task orchestartor.
//
// Written for this SO question: http://stackoverflow.com/q/28334314/796832
// Gist: https://gist.github.com/MadLittleMods/d4083d2ba35e2f850161
//
// Params:
//      taskNames: string or array of strings of task names
//      debugLog: *optional* boolean to print some debug information to the console
function gulpRunTasksAndGetStreams(taskNames, /*optional*/debugLog) {
    // You can pass in a single task or an array of tasks to complete
    taskNames = [].concat(taskNames);

    // We polyfill the pieces of `gulp-util` that we use in case some one wants to use it without installing `gulp-util`
    var gutil;
    try {
        gutil = require('gulp-util');
    }
    catch(err) {
        gutil = {
            log: console.log,
            colors: {
                cyan: function(str) {
                    return str;
                },
                magenta: function(str) {
                    return str;
                }
            }
        };
    }

    var resultantTaskInfo = [];
    var taskMap = gulp.tasks;

    // Satisfy all of the task dependencies, create a placeholder stream, and collect the func 
    // to make the real stream to feed in later when the dependencies are done `mergedDepStream.on('end')`
    var mergedDepStream = null;
    taskNames.forEach(function(taskName) {
        var task = taskMap[taskName];

        if(debugLog) {
            gutil.log('root task:', gutil.colors.cyan(taskName), 'started working');
        }

        // Run any dependencies first
        var depStreamResult = runDependenciesRecursivelyForTask(taskName, taskMap);

        if(depStreamResult) {
            mergedDepStream = mergedDepStream ? es.merge(mergedDepStream, depStreamResult) : depStreamResult;
        }

        if(debugLog) {
            if(depStreamResult) {
                depStreamResult.on('end', function() {
                    gutil.log('root task:', gutil.colors.cyan(taskName), 'deps done');
                });
            }
            else {
                gutil.log('root task:', gutil.colors.cyan(taskName), 'no deps present');
            }
        }

        // Then push the task itself onto the list
        resultantTaskInfo.push({
            stream: es.through(),
            fn: task.fn
        });
    });

    // Once all of the dependencies have completed
    mergedDepStream.on('end', function() {

        if(debugLog) {
            gutil.log('All dependencies done, piping in real root tasks');
        }

        // Pipe the actual task into our placeholder
        resultantTaskInfo.forEach(function(taskInfo) {
            var actualTaskStream = taskInfo.fn();
            actualTaskStream.pipe(taskInfo.stream);
        });
    });


    // Recursively gets all of dependencies for a task in order
    function runDependenciesRecursivelyForTask(taskName, taskMap, mergedDependencyStream) {
        var task = taskMap[taskName];

        task.dep.forEach(function(depTaskName) {
            var depTask = taskMap[depTaskName];
            if(debugLog) {
                gutil.log('dep task:', gutil.colors.cyan(depTaskName), 'started working');
            }

            // Dependencies can have dependencies
            var recursiveStreamResult = null;
            if(depTask.dep.length) {
                recursiveStreamResult = runDependenciesRecursivelyForTask(depTaskName, taskMap, mergedDependencyStream);
                mergedDependencyStream = mergedDependencyStream ? es.merge(mergedDependencyStream, recursiveStreamResult) : recursiveStreamResult;
            }

            if(depTask.fn) {
                var whenStreamHandledCallback = function(/* we only use `noDeps` for logging */noDeps) {
                    if(debugLog) {
                        if(!noDeps) {
                            gutil.log('dep task:', gutil.colors.cyan(depTask.name), 'deps done');
                        }
                        else {
                            gutil.log('dep task:', gutil.colors.cyan(depTask.name), 'no deps present');
                        }
                    }

                    var depTaskStream = depTask.fn();
                    // Merge it in overall dependency progress stream
                    mergedDependencyStream = mergedDependencyStream ? es.merge(mergedDependencyStream, depTaskStream) : depTaskStream;
                };

                if(recursiveStreamResult === null) {
                    whenStreamHandledCallback(true);
                }
                else {
                    recursiveStreamResult.on('end', whenStreamHandledCallback);
                }
            }
        });

        return mergedDependencyStream;
    }


    // Return the (placeholder) streams which will get piped the real stream once the dependencies are done
    return resultantTaskInfo.map(function(taskInfo) {
        return taskInfo.stream;
    });
}

Answer 2

@MLM had the right idea with about combining streams.

But don't forget that Gulp's just Javascript.

Try this:

const merge = require('merge-stream');
// or pick an alternative stream-merge library: 
//const merge = require('event-stream').merge; //1317 stars on GitHub
//const merge = require('merge-stream');       //102 stars on GitHub
//const merge = require('merge2');             //75 stars on GitHub
//const merge = require('stream-series');      //23 stars on GitHub, keeps events in order
//const merge = require('gulp-merge');         //renamed to merge2
/*const merge = require('streamqueue')         //54 stars on GitHub
    .bind(null, {objectMode: true}); //required for streamqueue vinyl streams
*/

function firstGroup() {
  return gulp.src('some/files/*.js')
    .pipe(doSomething())
    .pipe(concat('some-scripts.js'));
}
gulp.task('firstGroup', funtion() {
  return firstGroup()
    .pipe(gulp.dest('dest'));
});

function secondGroup() {
  return gulp.src('some/other/files/*.js')
    .pipe(doSomethingElse())
    .pipe(concat('some-other-scripts.js'));
}
gulp.task('secondGroup', function() {
  return secondGroup()
    .pipe(gulp.dest('dest'));
});

gulp.task('allScripts', function() {
  return merge(firstGroup(), secondGroup())
    .pipe(concat('all-scripts.js'))
    .pipe(gulp.dest('dest'))
});

And probably name your tasks and their related functions a bit better than above.

That being said, it's probably still easier and more clear to delete the files in the last task.

var del = require('del');

gulp.task('allScripts', ['firstGroup','secondGroup'], function(done) {
  var intermediariesGlob = ['dest/some-scripts.js','dest/some-other-scripts.js'];
  gulp.src(intermediariesGlob)
    .pipe(concat('all-scripts.js'))
    .pipe(gulp.dest('dest'))
    .on('end', function() {
      del(intermediariesGlob)
        .then(function() {done();}); //don't just then(done), the array returned by the promise will become the error parameter of done
    });
});