You could wrap your router.js in a function that accepts the passport object as parameter and handles the instantiation of the router also. Then, module.export that function instead of only the router.
EDIT: including example
server.js:
var passport = require('passport');
var router = require('./app/routes')(app, passport);
app.use('/', router);
app.listen(8080);
routes.js:
var express = require('express');
module.exports = function(app, passport){
var router = express.Router();
// routes go here
// do stuff with passport
return router;
}