If you choose to set a reference to FSO inside your function, then in each recursion will be used new FSO object.
Using single FSO object (either global, or passed) is quite enough. At least I don't know any benefit of using multiple FSO instances.
[EDIT] I appreciate @AnsgarWiechers comment, and to make the code ready for re-using, while kept the FSO out of the function, we can wrap our function in a class.
With New FileInfo
WScript.Echo .FileSize("C:\temp", "txt", True)
End With
Class FileInfo
Private m_oFSO
Public Function FileSize(sRootDir, sExtension, bRecursive)
Dim oFolder, oFile, sFExt
sFExt = LCase(sExtension)
Set oFolder = m_oFSO.GetFolder(sRootDir)
For Each oFile In oFolder.Files
If LCase(m_oFSO.GetExtensionName(oFile.Name)) = sFExt Then
FileSize = FileSize + oFile.Size
End If
Next
If bRecursive Then
Dim oSubFolder
For Each oSubFolder In oFolder.SubFolders
FileSize = FileSize + FileSize(oSubFolder, sExtension, True)
Next
End If
End Function
Private Sub Class_Initialize
Set m_oFSO = CreateObject("Scripting.FileSystemObject")
End Sub
Private Sub Class_Terminate
Set m_oFSO = Nothing
End Sub
End Class