Persistent Logging in Windows Azure

Question

So far I have tried a couple of different strategies to persist my logs into my storage area.

Trace.Listeners.Add(new TextWriterTraceListener(new Logger(true).output));
Trace.WriteLine("Added a persistant logging listener in Action Worker.");

And here is a Logger:

 public class Logger 
    {
        private string PersistedLogArea = "CustomLoggingLocation";
        public StreamWriter output;

        public Logger(int hours, int minutes, int ms)
        {
            string loglocation = CreatePersistedLogfile();
            FileStream fs = new FileStream(loglocation, FileMode.OpenOrCreate, FileAccess.ReadWrite);
            output = new StreamWriter(fs);
            InitializeWadConfiguration(hours, minutes, ms);
            output.Write("If we shadows have offended (persistant logging initialized)");

        }
        public Logger(bool interactive)
        {
            string loglocation = CreatePersistedLogfile();
            FileStream fs = new FileStream(loglocation, FileMode.OpenOrCreate, FileAccess.ReadWrite);
            output = new StreamWriter(fs);
            InitializeWadConfiguration(0, 1, 0);
            output.Write("If we shadows have offended (persistant logging initialized)");
        }



        private string CreatePersistedLogfile()
        {
            LocalResource localResource =
              RoleEnvironment.GetLocalResource("CustomLoggingLocation");
            String logPath =
              Path.Combine(localResource.RootPath, "Logs");
            String fileName =
              Path.Combine(logPath, Path.GetRandomFileName());

            if (!Directory.Exists(logPath))
            {
                Directory.CreateDirectory(logPath);
            }
            return fileName;

        }

        private void InitializeWadConfiguration(int hours, int minutes, int ms)
        {
            //String wadConnectionString =
             //"Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString";
            String customContainerName = "persisted-logfiles";

            DiagnosticMonitorConfiguration dmc =
             DiagnosticMonitor.GetDefaultInitialConfiguration();

            LocalResource localResource =
             RoleEnvironment.GetLocalResource(PersistedLogArea);
            String logPath =
             Path.Combine(localResource.RootPath, "Logs", "logfile.txt");

            DirectoryConfiguration directoryConfiguration =
             new DirectoryConfiguration()
             {
                 Container = customContainerName,
                 DirectoryQuotaInMB =
                  localResource.MaximumSizeInMegabytes,
                 Path = logPath
             };
            dmc.Directories.DataSources.Add(directoryConfiguration);
            if (ms > 0)
            {
                dmc.Directories.ScheduledTransferPeriod = TimeSpan.FromMilliseconds(ms);
            }
            else if (minutes > 0)
            {
                dmc.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(minutes);
            }
            else if (hours > 0)
            {
                dmc.Directories.ScheduledTransferPeriod =
                 TimeSpan.FromHours(hours);
            }
            dmc.Logs.BufferQuotaInMB = 100;
            dmc.Logs.ScheduledTransferPeriod = TimeSpan.FromHours(1);
            dmc.Logs.ScheduledTransferLogLevelFilter =
             LogLevel.Verbose;

            string connectionString = "DefaultEndpointsProtocol=https;AccountName=mitimesexchangeapp;AccountKey=f7hU2fhpHTgswXShB67MALNWVD2CH4jF+c7Zuw+rDU1U1ZFJoyuPS9BYojAwLxE3oFMEL281i/SYb8YhDbmsSg==";

            Microsoft.WindowsAzure.CloudStorageAccount cloudStorageAccount = Microsoft.WindowsAzure.CloudStorageAccount.Parse(connectionString);
            DiagnosticMonitor.Start(cloudStorageAccount, dmc);
        }
}

I run this code and I see nothing in the storage table. It's super disheartening.

So I decided to try some config based listener attachment and I did all the things here.

diagnostics.wadcfg:

<?xml version="1.0" encoding="utf-8" ?>
<DiagnosticMonitorConfiguration xmlns="http://schemas.microsoft.com/ServiceHosting/2010/10/DiagnosticsConfiguration"
      configurationChangePollInterval="PT1M"
      overallQuotaInMB="4096">

  <DiagnosticInfrastructureLogs bufferQuotaInMB="0"
     scheduledTransferLogLevelFilter="Verbose"
     scheduledTransferPeriod="PT30M" />

  <Logs bufferQuotaInMB="0"
     scheduledTransferLogLevelFilter="Verbose"
     scheduledTransferPeriod="PT30M" />

  <Directories bufferQuotaInMB="0"
     scheduledTransferPeriod="PT30M">
    <!-- FailedRequestLogs and IISLogs are only relevant to Web roles -->
    <CrashDumps container="wad-crash-dumps" directoryQuotaInMB="0" />
    <FailedRequestLogs container="wad-frq" directoryQuotaInMB="0" />
    <IISLogs container="wad-iis" directoryQuotaInMB="0" />
    <DataSources>
      <DirectoryConfiguration container="diagnostics-custom-logs" directoryQuotaInMB="1024">
        <LocalResource name="MyCustomLogs" relativePath="."/>
      </DirectoryConfiguration>
    </DataSources>
  </Directories>

  <PerformanceCounters bufferQuotaInMB="0" scheduledTransferPeriod="PT30M">
    <PerformanceCounterConfiguration counterSpecifier="\Memory\Available Bytes" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\Processor(_Total)\% Processor Time" sampleRate="PT30S" />
    <!-- These three elements are only relevant to Web roles 
    <PerformanceCounterConfiguration counterSpecifier="\Process(w3wp)\% Processor Time" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\Process(w3wp)\Private Bytes" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\Process(w3wp)\Thread Count" sampleRate="PT30S" />-->
    <!-- These three elements are only relevant to Worker roles -->
    <PerformanceCounterConfiguration counterSpecifier="\Process(WaWorkerHost)\% Processor Time" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\Process(WaWorkerHost)\Private Bytes" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\Process(WaWorkerHost)\Thread Count" sampleRate="PT30S" />

    <PerformanceCounterConfiguration counterSpecifier="\.NET CLR Interop(_Global_)\# of marshalling" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\.NET CLR Loading(_Global_)\% Time Loading" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\.NET CLR LocksAndThreads(_Global_)\Contention Rate / sec" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\.NET CLR Memory(_Global_)\# Bytes in all Heaps" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\.NET CLR Networking(_Global_)\Connections Established" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\.NET CLR Remoting(_Global_)\Remote Calls/sec" sampleRate="PT30S" />
    <PerformanceCounterConfiguration counterSpecifier="\.NET CLR Jit(_Global_)\% Time in Jit" sampleRate="PT30S" />
  </PerformanceCounters>

  <WindowsEventLog bufferQuotaInMB="0"
     scheduledTransferLogLevelFilter="Verbose"
     scheduledTransferPeriod="PT30M">
    <DataSource name="Application!*" />
    <DataSource name="System!*" />
  </WindowsEventLog>

  <Logs bufferQuotaInMB="0"
         scheduledTransferLogLevelFilter="Verbose"
         scheduledTransferPeriod="PT30M" />

  <DiagnosticInfrastructureLogs bufferQuotaInMB="0"
                                  scheduledTransferLogLevelFilter="Verbose"
                                  scheduledTransferPeriod="PT30M" />

  <Directories bufferQuotaInMB="0" scheduledTransferPeriod="PT30M">
    <DataSources>
      <DirectoryConfiguration container="diagnostics-custom-logs" directoryQuotaInMB="128">
        <LocalResource name="CustomLogLocation" relativePath="." />
      </DirectoryConfiguration>
    </DataSources>
  </Directories>
</DiagnosticMonitorConfiguration>

Config:

<ConfigurationSettings>
  <Setting name="Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString" value="DefaultEndpointsProtocol=https;AccountName=<name>;AccountKey=<key>"/>
</ConfigurationSettings>

Let me know if there is anything else you care to see. I would REALLY like my logs to persist into my storage account.

EDIT:

It was working the whole time. I needed the storage explorer to see the information, I didn't understand it was going to a table. Reading comprehension for the win.

Answer 1

If you want to log your Trace output you need to use the TraceListener provided by Azure SDK.

Something like this in your web.config file

<listeners>
    <add type="Microsoft.WindowsAzure.Diagnostics.DiagnosticMonitorTraceListener, Microsoft.WindowsAzure.Diagnostics, Version=1.8.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35"
         name="AzureDiagnostics">
        <filter type="" />
    </add>
</listeners>

or register the listener using the code.

Once configured all Trace.Write... messages would go into WADLogsTable at predefined intervals. And this does not require any custom code.

Hope you had looked at this msdn link