feat(web-server): cache preprocessed files

vojtajina · vojtajina · commit c786ee2ea19d · 2013-11-30T22:19:00.000-08:00
Now Karma does not write preprocessed files into disk (as the web server would read them) and store that in memory. Especially when running with multiple browsers (each browser has to fetch the file), this saves FS access (1 write per preprocessed file; x reads per preprocessed file, where x is number of captured browsers).

This is in trade for slightly higher memory usage. I however tried AngularJS with this change (forcing all the files to be processed/cached) and the memory consumption was not significantly higher.

We can optimize this to clean up the cache, especially if there is only a single browser captured
diff --git a/lib/middleware/common.js b/lib/middleware/common.js
@@ -26,11 +26,26 @@ var serve404 = function(response, path) {
 
 
 var createServeFile = function(fs, directory) {
-  return function(filepath, response, transform) {
+  return function(filepath, response, transform, content) {
+    var responseData;
+
     if (directory) {
       filepath = directory + filepath;
     }
 
+    // serve from cache
+    if (content) {
+      response.setHeader('Content-Type', mime.lookup(filepath, 'text/plain'));
+
+      // call custom transform fn to transform the data
+      responseData = transform && transform(content) || content;
+
+      response.writeHead(200);
+
+      log.debug('serving (cached): ' + filepath);
+      return response.end(responseData);
+    }
+
     return fs.readFile(filepath, function(error, data) {
       if (error) {
         return serve404(response, filepath);
@@ -39,7 +54,7 @@ var createServeFile = function(fs, directory) {
       response.setHeader('Content-Type', mime.lookup(filepath, 'text/plain'));
 
       // call custom transform fn to transform the data
-      var responseData = transform && transform(data.toString()) || data;
+      responseData = transform && transform(data.toString()) || data;
 
       response.writeHead(200);
 
diff --git a/lib/middleware/source-files.js b/lib/middleware/source-files.js
@@ -37,15 +37,15 @@ var createSourceFilesMiddleware = function(filesPromise, serveFile,
       var file = findByPath(files.served, requestedFilePath);
 
       if (file) {
-        serveFile(file.contentPath, response, function() {
+        serveFile(file.contentPath || file.path, response, function() {
           if (/\?\d+/.test(request.url)) {
             // files with timestamps - cache one year, rely on timestamps
             common.setHeavyCacheHeaders(response);
           } else {
             // without timestamps - no cache (debug)
             common.setNoCacheHeaders(response);
           }
-        });
+        }, file.content);
       } else {
         next();
       }
diff --git a/lib/preprocessor.js b/lib/preprocessor.js
@@ -1,13 +1,9 @@
-var path = require('path');
 var fs = require('graceful-fs');
 var crypto = require('crypto');
 var mm = require('minimatch');
 
 var log = require('./logger').create('preprocess');
 
-// TODO(vojta): extract get/create temp dir somewhere else (use the same for launchers etc)
-var TMP = process.env.TMPDIR || process.env.TMP || process.env.TEMP || '/tmp';
-
 var sha1 = function(data) {
   var hash = crypto.createHash('sha1');
   hash.update(data);
@@ -23,10 +19,9 @@ var createPreprocessor = function(config, basePath, injector) {
     var preprocessors = [];
     var nextPreprocessor = function(content) {
       if (!preprocessors.length) {
-        file.contentPath = path.normalize(TMP + '/'  + sha1(file.path) + path.extname(file.path));
-        return fs.writeFile(file.contentPath, content, function() {
-          done();
-        });
+        file.contentPath = null;
+        file.content = content;
+        return done();
       }
 
       preprocessors.shift()(content, file, nextPreprocessor);
diff --git a/test/unit/middleware/source-files.spec.coffee b/test/unit/middleware/source-files.spec.coffee
@@ -12,6 +12,7 @@ describe 'middleware.source-files', ->
     base:
       path:
         'a.js': mocks.fs.file(0, 'js-src-a')
+        'index.html': mocks.fs.file(0, '<html>')
     src:
       'some.js': mocks.fs.file(0, 'js-source')
     'utf8ášč':
@@ -133,3 +134,30 @@ describe 'middleware.source-files', ->
       done()
 
     callHandlerWith '/base/some.js'
+
+  it 'should set content-type headers', (done) ->
+    servedFiles [
+      new File('/base/path/index.html')
+    ]
+
+    response.once 'end', ->
+      expect(response._headers['Content-Type']).to.equal 'text/html'
+      done()
+
+    callHandlerWith '/base/index.html'
+
+
+  it 'should use cached content if available', (done) ->
+    cachedFile = new File('/some/file.js')
+    cachedFile.content = 'cached-content'
+
+    servedFiles [
+      cachedFile
+    ]
+
+    response.once 'end', ->
+      expect(nextSpy).not.to.have.been.called
+      expect(response).to.beServedAs 200, 'cached-content'
+      done()
+
+    callHandlerWith '/absolute/some/file.js'
diff --git a/test/unit/preprocessor.spec.coffee b/test/unit/preprocessor.spec.coffee
@@ -11,20 +11,12 @@ describe 'preprocessor', ->
     mockFs = mocks.fs.create
       some:
         'a.js': mocks.fs.file 0, 'content'
-        'style.less': mocks.fs.file 0, 'whatever'
-      temp: {} # so that we can write preprocessed content here
-
 
     mocks_ =
       'graceful-fs': mockFs
       minimatch: require 'minimatch'
 
-    globals_ =
-      process:
-        env: TMPDIR: '/temp'
-        nextTick: process.nextTick
-
-    m = mocks.loadFile __dirname + '/../../lib/preprocessor.js', mocks_, globals_
+    m = mocks.loadFile __dirname + '/../../lib/preprocessor.js', mocks_
 
 
   it 'should preprocess matching file', (done) ->
@@ -40,7 +32,7 @@ describe 'preprocessor', ->
     pp file, ->
       expect(fakePreprocessor).to.have.been.called
       expect(file.path).to.equal 'path-preprocessed'
-      expect(mockFs.readFileSync(file.contentPath).toString()).to.equal 'new-content'
+      expect(file.content).to.equal 'new-content'
       done()
 
 
@@ -80,22 +72,5 @@ describe 'preprocessor', ->
       expect(fakePreprocessor1).to.have.been.calledOnce
       expect(fakePreprocessor2).to.have.been.calledOnce
       expect(file.path).to.equal 'path-p1-p2'
-      expect(mockFs.readFileSync(file.contentPath).toString()).to.equal 'content-c1-c2'
-      done()
-
-
-  it 'should keep processed extension', (done) ->
-    fakePreprocessor = sinon.spy (content, file, done) ->
-      file.path = file.path.replace '.less', '.css'
-      done content
-
-    injector = new di.Injector [{
-      'preprocessor:less': ['factory', -> fakePreprocessor]
-    }]
-
-    pp = m.createPreprocessor {'**/*.less': ['less']}, null, injector
-    file = {originalPath: '/some/style.less', path: '/some/style.less'}
-
-    pp file, ->
-      expect(file.contentPath).to.match /\.css$/
+      expect(file.content).to.equal 'content-c1-c2'
       done()
