1. Sitecore Support Avatar Sitecore Support
  2. Untitled project
  3. Sitecore.Support.ParallelEventQueue

Wiki

Clone wiki

Sitecore.Support.ParallelEventQueue / Home

View History

Sitecore.Support.ParallelEventQueue

Tweak to process EventQueue from Web database with several parallel threads which boosts the performance. It can be configured to write detailed logs into History table.

Dependencies

This extension depends on Sitecore.Abstract.DataProviders, Sitecore.AdvancedSettings and Sitecore.AdvancedHistory.

Deployment

In order to deploy the tool:

  1. download and extract the Sitecore.Support.ParallelEventQueue-1.0.6.0.zip file into Website folder of Content-Delivery instances in solution.

Contents

bin\Sitecore.Abstract.DataProviders.dll

The assembly allows to specify custom SqlServerEventQueue type without customizing SqlServerDataProvider.

bin\Sitecore.Support.ParallelEventQueue.dll

The assembly contains ParallelEventQueueWithSeparateQueues and ParallelEventQueueWithSingleSharedQueue implementations of parallel queues, which are almost the same with minor differences. When processing events, main thread writes following data to the log file:

ParallelEventQueue.MainThread.Count: 213
ParallelEventQueue.MainThread.Time.Total: 200ms
ParallelEventQueue.MainThread.Time.Read: 190ms

Each particular thread writes its own logs:

ParallelEventQueue.WorkerThread{1}.Count: 123
ParallelEventQueue.WorkerThread{1}.PublishEndCount: 1
ParallelEventQueue.WorkerThread{1}.Time.Total: 2222ms 
ParallelEventQueue.WorkerThread{1}.Time.Deserialize: 150ms
ParallelEventQueue.WorkerThread{1}.Time.Process: 1900ms
ParallelEventQueue.WorkerThread{1}.Time.PublishEndSleep: 2000ms

App_Config\Include\Sitecore.Abstract.DataProviders.config

The config file allows to specify custom SqlServerEventQueue type without customizing SqlServerDataProvider.

#!xml

<configuration xmlns:patch="http://www.sitecore.net/xmlconfig/">
  <sitecore>
    <dataProviders>
      <main patch:instead="main" type="Sitecore.Abstract.Data.SqlServer.SqlServerDataProvider, Sitecore.Abstract.DataProviders">
        <param connectionStringName="$(1)" />
        <param desc="eventQueueType"></param>
        <param desc="publishQueueProviderType"></param>
        <param desc="databasePropertiesProviderType"></param>
        <Name>$(1)</Name>
      </main>
    </dataProviders>
  </sitecore>
</configuration>

App_Config\Include\Sitecore.AdvancedHistory.config

#!xml

<configuration xmlns:patch="http://www.sitecore.net/xmlconfig/" xmlns:set="http://www.sitecore.net/xmlconfig/set/">
  <sitecore>
    <advancedHistoryManager defaultProvider="redirecting">
      <providers>
        <clear />
        <!--        
        <add name="sql" type="Sitecore.AdvancedHistory.SqlServer.SqlServerAdvancedHistoryProvider, Sitecore.AdvancedHistory" database="core" />        
        <add name="csv" type="Sitecore.AdvancedHistory.CsvFile.CsvFileAdvancedHistoryProvider, Sitecore.AdvancedHistory" database="master" filePath="$(dataFolder)\History\web.{0:yyyyMMdd}.csv" fileLifeTime="1.00:00:00" commitInterval="00:00:10" />
        <add name="log" type="Sitecore.AdvancedHistory.LogFile.LogFileAdvancedHistoryProvider, Sitecore.AdvancedHistory" database="web" />

        <add name="switcher" type="Sitecore.AdvancedHistory.SwitchingAdvancedHistoryProvider, Sitecore.AdvancedHistory" database="*" />
        -->

        <add name="sql" type="Sitecore.AdvancedHistory.SqlServer.SqlServerAdvancedHistoryProvider, Sitecore.AdvancedHistory" database="web" />
        <add name="csv" type="Sitecore.AdvancedHistory.CsvFile.CsvFileAdvancedHistoryProvider, Sitecore.AdvancedHistory" database="web" filePath="$(dataFolder)\History\web.{0:yyyyMMdd}.csv" fileLifeTime="1.00:00:00" commitInterval="00:00:10" />
        <add name="log" type="Sitecore.AdvancedHistory.LogFile.LogFileAdvancedHistoryProvider, Sitecore.AdvancedHistory" database="web" />

        <add name="redirecting" type="Sitecore.AdvancedHistory.SettingBasedAdvancedHistoryProvider, Sitecore.AdvancedHistory" database="*" />
      </providers>
    </advancedHistoryManager>
    <settings>
      <setting name="AdvancedHistory.Provider" value="sql" />
    </settings>
  </sitecore>
</configuration>

App_Config\Include\Sitecore.AdvancedSettings.config

#!xml

<configuration xmlns:set="http://www.sitecore.net/xmlconfig/set/">
  <sitecore>
    <hooks>
      <hook type="Sitecore.AdvancedSettingsInitializeHook, Sitecore.AdvancedSettings" />
    </hooks>
    <settings>
      <!--  SYSTEM DATABASE
            Sets the name of system database to poll settings values from.
      -->
      <setting name="AdvancedSettings.SystemDatabase" value="core" />

      <!--  POLLING INTERVAL 
            Sets how often events should be checked in the databases.
      -->
      <setting name="AdvancedSettings.PollingInterval" value="00:00:10" />
    </settings>
  </sitecore>
</configuration>

App_Config\Include\Sitecore.Support.ParallelEventQueue.CommonSettings.config

#!xml

<configuration xmlns:patch="http://www.sitecore.net/xmlconfig/">
  <sitecore>
    <settings>
      <!--  DATABASE NAME 
            DatabaseName indicates the database where parallel execution is required.

            Default: web
      -->
      <setting name="ParallelEventQueue.DatabaseName" value="web" />

      <!--  EVENT QUEUE - PARALLEL THREADS COUNT
            ParallelThreadsCount indicates total number of parallel threads that process in-memory queue that is being populated each 2 seconds from the database EventQueue table.

            Default: 4
      -->
      <setting name="ParallelEventQueue.ParallelThreadsCount" value="4" />

      <!--  EVENT QUEUE - HISTORY ENABLED
            When enabled, a history entry is created in the web database for each event that is processed.

            Default: false
      -->
      <setting name="ParallelEventQueue.HistoryEnabled" value="false" />

      <!--  EVENT QUEUE - HISTORY STATS ENABLED
            When enabled, an Event Queue processing statistics is written into history table in the web database.  
            It is not taken into account when ParallelEventQueue.HistoryEnabled is disabled.

            Default: true
      -->
      <setting name="ParallelEventQueue.HistoryStatsEnabled" value="true" />

      <!--  EVENT QUEUE - SECURITY DISABLER
            When enabled, a security disabler is used for processing all events.

            Default: false
      -->
      <setting name="ParallelEventQueue.SecurityDisabler" value="false" />

      <!--  EVENT QUEUE THREAD - BATCH SIZE
            BatchSize indicates maximum number of events processed one-by-one before writing to log file and trying a deep sleep.

            Default: 1000
      -->
      <setting name="ParallelEventQueue.EventQueueThread.BatchSize" value="1000" />

      <!--  EVENT QUEUE THREAD - DEEP SLEEP
            DeepSleep is intended to save CPU when no publish activity happen.
            After 1000 (BatchSize) attempts to get an event from empty queue system will fall asleep for 1 second before next 1000 attempts. 
            Increase this value if see high CPU utilization or when notice too many log entries "Health.EQ{1}.Count: 0"

            Default: 1000 (1 second)
      -->
      <setting name="ParallelEventQueue.EventQueueThread.DeepSleep" value="1000" />

      <!--  EVENT QUEUE THREAD - LOG INTERVAL
            LogInterval defines minimal interval between log entries generated by events processing threads.

            Default: 00:05:00
      -->
      <setting name="ParallelEventQueue.EventQueueThread.LogInterval" value="00:05:00" />

      <!--  EVENT QUEUE THREAD - PUBLISH END SLEEP
            PublishEndSleep defines sleep interval between checks if it is a right time to process publish:end:remote event or it is necessary to wait.
            In order to keep indexes consistent and healthy, publish:end:remote must be processed after all the item:*:remote events located in the queue
            before it (publish:end:remote will have greater timestamp value than item:*:remote ones). So as soon as one of the threads receives such an 
            event, the thread will check if other threads process events with greater timestamps rather than this one, otherwise it will wait 1 second
            (which is controlled by this setting) before checking this again.

            Default: 1000 (1 second)
      -->
      <setting name="ParallelEventQueue.EventQueueThread.PublishEndSleep" value="1000" />

      <!--  MAIN THREAD - LOG INTERVAL
            LogInterval defines minimal interval between log entries generated by main thread that reads data from the EventQueue table.

            Default: 00:05:00
      -->
      <setting name="ParallelEventQueue.MainThread.LogInterval" value="00:05:00" />
    </settings>
  </sitecore>
</configuration>

App_Config\Include\Sitecore.Support.ParallelEventQueue.SeparateQueues.config

#!xml

<configuration xmlns:patch="http://www.sitecore.net/xmlconfig/">
  <sitecore>
    <dataProviders>
      <main>
        <param desc="eventQueueType">Sitecore.Support.Data.Eventing.ParallelEventQueueWithSeparateQueues, Sitecore.Support.ParallelEventQueue</param>
      </main>
    </dataProviders>
    <eventing>
      <providers>
        <clear />
        <add name="sitecore" set:type="Sitecore.Support.Eventing.ParallelEventProvider, Sitecore.Support.ParallelEventQueue" />
      </providers>
    </eventing>
  </sitecore>
</configuration>

App_Config\Include\Sitecore.Support.ParallelEventQueue.SingleSharedQueue.config.disabled

#!xml

<configuration xmlns:patch="http://www.sitecore.net/xmlconfig/">
  <sitecore>
    <dataProviders>
      <main>
        <param desc="eventQueueType">Sitecore.Support.Data.Eventing.ParallelEventQueueWithSingleSharedQueue, Sitecore.Support.ParallelEventQueue</param>
      </main>
    </dataProviders>
    <eventing>
      <providers>
        <clear />
        <add name="sitecore" set:type="Sitecore.Support.Eventing.ParallelEventProvider, Sitecore.Support.ParallelEventQueue" />
      </providers>
    </eventing>
  </sitecore>
</configuration>

Updated